* Embed nginx inside Docker image so all services are reversed proxied behind port 80

* Use Varnish 3 from source tarball, newer and OS Varnish could not disable telnet authentication required by Carto
* Remove sources of varnish, gdal, /cartodb/.git when no longer needed in order to make image smaller
* Dropped Nokia and GMAP basemaps, they did not work
* Dropped schema trigger installation, it is no longer needed
* Configure Carto in Python and NodeJS
* Build assets
* Use self as asset_host, assets are hosted by nginx instead of rails
This commit is contained in:
Stefan Verhoeven 2017-04-03 22:46:52 +02:00
parent 3397aed11d
commit bbf2c54325
7 changed files with 133 additions and 99 deletions

View File

@ -38,6 +38,7 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
libproj-dev \
gdal-bin \
libgdal1-dev \
libgdal-dev \
postgresql-9.5 \
postgresql-client-9.5 \
postgresql-contrib-9.5 \
@ -47,11 +48,11 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
postgresql-9.5-postgis-2.2 \
postgresql-9.5-postgis-scripts \
postgis \
liblwgeom-2.2-5 \
ca-certificates \
redis-server \
python2.7-dev \
python-setuptools \
varnish \
imagemagick \
libmapnik-dev \
mapnik-utils \
@ -59,6 +60,9 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
python-argparse \
python-gdal \
python-chardet \
python-pip \
python-all-dev \
python-docutils \
openssl \
libreadline6 \
zlib1g \
@ -86,51 +90,64 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
libgif-dev \
libgmp-dev \
libicu-dev \
wget \
nginx-light \
net-tools \
--no-install-recommends && \
rm -rf /var/lib/apt/lists/*
RUN git config --global user.email you@example.com
RUN git config --global user.name "Your Name"
# Varnish 3, Ubuntu:16.04 comes with Varnish 4.1 which can't be run with anonymous admin telnet
RUN cd /opt && \
wget https://repo.varnish-cache.org/source/varnish-3.0.7.tar.gz && \
tar -zxf varnish-3.0.7.tar.gz && \
cd varnish-3.0.7 && \
./configure --prefix=/opt/varnish && \
make && \
make install &&
cd /opt &&
rm -rf varnish-3.0.7 varnish-3.0.7.tar.gz
# ogr2ogr2 static build, see https://github.com/CartoDB/cartodb/wiki/How-to-build-gdal-and-ogr2ogr2
# using cartodb instruction got error https://trac.osgeo.org/gdal/ticket/6073
# https://github.com/OSGeo/gdal/compare/trunk...CartoDB:ogr2ogr2 has no code changes, so just use latest gdal tarball
RUN cd /opt && \
curl http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz -o gdal-2.1.1.tar.gz && \
tar -zxf gdal-2.1.1.tar.gz && \
cd gdal-2.1.1 && \
./configure --disable-shared && \
make -j 4 && \
cp apps/ogr2ogr /usr/bin/ogr2ogr2 && \
rm -rf /opt/ogr2ogr2 /root/.gitconfig
curl http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz -o gdal-2.1.1.tar.gz && \
tar -zxf gdal-2.1.1.tar.gz && \
cd gdal-2.1.1 && \
./configure --disable-shared && \
make -j 4 && \
cp apps/ogr2ogr /usr/bin/ogr2ogr2 && \
cd /opt && \
rm -rf /opt/ogr2ogr2 /opt/gdal-2.1.1.tar.gz /root/.gitconfig /opt/gdal-2.1.1
# Install NodeJS
RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr
RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr && \
npm install -g grunt-cli && \
npm install -g npm@~2.14.0
# Install rvm
RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3
RUN curl -L https://get.rvm.io | bash -s stable --ruby
RUN echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc
RUN /bin/bash -l -c rvm requirements
ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
RUN echo rvm_max_time_flag=15 >> ~/.rvmrc
RUN /bin/bash -l -c 'rvm install 2.2.3'
RUN /bin/bash -l -c 'rvm use 2.2.3 --default'
RUN /bin/bash -l -c 'gem install bundle archive-tar-minitar'
# Install bundler
RUN /bin/bash -l -c 'gem install bundler --no-doc --no-ri'
RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 && \
curl -L https://get.rvm.io | bash -s stable --ruby && \
echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc && \
/bin/bash -l -c rvm requirements
ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \
RUN echo rvm_max_time_flag=15 >> ~/.rvmrc && \
/bin/bash -l -c 'rvm install 2.2.3' && \
/bin/bash -l -c 'rvm use 2.2.3 --default' && \
/bin/bash -l -c 'gem install bundle archive-tar-minitar' && \
/bin/bash -l -c 'gem install bundler compass --no-doc --no-ri'
# Setting PostgreSQL
RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf
RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf && \
service postgresql start && \
createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && \
createuser tileuser --no-createrole --no-createdb --no-superuser -U postgres && \
service postgresql stop
# Install schema_triggers
RUN git clone https://github.com/CartoDB/pg_schema_triggers.git && \
cd pg_schema_triggers && \
make all install && \
sed -i \
"/#shared_preload/a shared_preload_libraries = 'schema_triggers.so'" \
/etc/postgresql/9.5/main/postgresql.conf
# Initialize template postgis db
ADD ./template_postgis.sh /tmp/template_postgis.sh
RUN service postgresql start && /bin/su postgres -c \
/tmp/template_postgis.sh && service postgresql stop
@ -138,6 +155,7 @@ RUN service postgresql start && /bin/su postgres -c \
# Install cartodb extension
RUN git clone https://github.com/CartoDB/cartodb-postgresql && \
cd cartodb-postgresql && \
git checkout master && \
PGUSER=postgres make install
ADD ./cartodb_pgsql.sh /tmp/cartodb_pgsql.sh
RUN service postgresql start && /bin/su postgres -c \
@ -145,23 +163,22 @@ RUN service postgresql start && /bin/su postgres -c \
# Install CartoDB API
RUN git clone git://github.com/CartoDB/CartoDB-SQL-API.git && \
cd CartoDB-SQL-API && ./configure && npm install
cd CartoDB-SQL-API && npm install
# Install Windshaft
RUN git clone git://github.com/CartoDB/Windshaft-cartodb.git && \
cd Windshaft-cartodb && ./configure && npm install && mkdir logs
cd Windshaft-cartodb && git checkout master && npm install && mkdir logs
# Install CartoDB (with the bug correction on bundle install)
RUN git clone git://github.com/CartoDB/cartodb.git && \
# Install CartoDB
RUN git clone --recursive git://github.com/CartoDB/cartodb.git && \
cd cartodb && \
perl -pi -e 's/jwt \(1\.5\.3\)/jwt (1.5.4)/' Gemfile.lock && \
/bin/bash -l -c 'bundle install' || \
/bin/bash -l -c "cd $(/bin/bash -l -c 'gem contents \
debugger-ruby_core_source' | grep CHANGELOG | sed -e \
's,CHANGELOG.md,,') && /bin/bash -l -c 'rake add_source \
VERSION=$(/bin/bash -l -c 'ruby --version' | awk \
'{print $2}' | sed -e 's,p55,-p55,' )' && cd /cartodb && \
/bin/bash -l -c 'bundle install'"
git checkout master && \
npm install && \
perl -pi -e 's/gdal==1\.10\.0/gdal==1.11.3/' python_requirements.txt && \
pip install --no-use-wheel -r python_requirements.txt && \
/bin/bash -l -c 'bundle install' && \
/bin/bash -l -c 'bundle exec grunt --environment development' && \
rm -rf .git
# Geocoder SQL client + server
RUN git clone https://github.com/CartoDB/data-services && \
@ -186,18 +203,19 @@ ADD ./config/app_config.yml /cartodb/config/app_config.yml
ADD ./config/database.yml /cartodb/config/database.yml
ADD ./create_dev_user /cartodb/script/create_dev_user
ADD ./setup_organization.sh /cartodb/script/setup_organization.sh
ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default
ADD ./config/varnish.vcl /etc/varnish.vcl
ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications
RUN service postgresql start && service redis-server start && \
bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && bash script/setup_organization.sh" && \
# Enable CARTO Builder
# bundle exec rake cartodb:features:enable_feature_for_all_users['editor-3'] && \
# bundle exec rake cartodb:features:enable_feature_for_all_users['explore_site']" && \
RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications && \
/opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl && \
service postgresql start && service redis-server start && \
bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && \
bash script/setup_organization.sh" && \
service postgresql stop && service redis-server stop
EXPOSE 3000 8080 8181
EXPOSE 80
ENV GDAL_DATA /usr/share/gdal/1.10
ENV GDAL_DATA /usr/share/gdal/1.11
ADD ./startup.sh /opt/startup.sh

View File

@ -20,22 +20,17 @@ How to build the container:
```
git clone https://github.com/sverhoeven/docker-cartodb.git
docker build -t="sverhoeven/cartodb" docker-cartodb/
docker build -t=sverhoeven/cartodb docker-cartodb/
```
How to run the container:
-------------------------
```
docker run -d -p 3000:3000 -p 8080:8080 -p 8181:8181 sverhoeven/cartodb
docker run -d -p 80:80 sverhoeven/cartodb
```
The ports the cartodb container publishes must be combined behind a single [NGINX](http://nginx.org/) web server.
In the GitHub repo there is an [example NGINX config file (config/cartodb.nginx.proxy.conf)](https://github.com/sverhoeven/docker-cartodb/blob/master/config/cartodb.nginx.proxy.conf), which needs to be added to /etc/nginx/conf.d/ directory, after which the web server must be restarted.
This will setup a reverse proxy for the CartoDB/imports (3000), SQL Api (8080) and Map api (8181) to default http port (80).
Alternativly use instructions at https://hub.docker.com/r/spawnthink/cartodb-nginx/ to run NGINX as a docker container with the correct config file already in it.
The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` to the machine where the web server is running.
The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` as the machine where the web server is running.
This can be done by adding cartodb.localhost alias to your hosts file. For example
```
sudo sh -c 'echo 127.0.1.1 cartodb.localhost >> /etc/hosts'

View File

@ -68,6 +68,8 @@ defaults: &defaults
app_keys:
editor: ''
embeds: ''
#fullstory:
#org: 'XXXXX'
facebook:
app_id: ''
admins: ''
@ -102,6 +104,15 @@ defaults: &defaults
wizard: ''
filter: ''
query: ''
applied_sql: ''
applied_cartocss: ''
modified_style_form: ''
completed_connection: ''
failed_connection: ''
created_analysis: ''
modified_analysis: ''
published_map: ''
exported_map: ''
mailing_track:
like_map: ''
trending_map: ''
@ -193,6 +204,7 @@ defaults: &defaults
bucket_name: ''
url_ttl: 7200
async_long_uploads: false
s3_endpoint: ''
uploads_path: 'public/uploads' # including 'uploads' assumes public path. Absolute path example: /tmp/exports/downloads
importer:
blacklisted_ip_addr: []
@ -209,6 +221,7 @@ defaults: &defaults
async_long_uploads: false
proxy_uri:
use_ssl: True
s3_endpoint: ''
unp_temporal_folder: '/tmp/imports/'
# It must end in `/uploads` and be accessible via HTTP, if relative will default to Rails.Root/#{uploads_path}
@ -265,12 +278,18 @@ defaults: &defaults
assets:
s3_bucket_name: "tests"
max_file_size: 5242880 # 5.megabytes
# Example for configuring organization assets.
# If 'aws.s3' configuration exists, S3 is used for storing assets and 'assets.organization.bucket' value is required. If 'aws.s3' config is not present, assets will be stored locally.
# organization:
# bucket: "tests" # Required if 'aws.s3' config is present. Bucket must exist beforehand. If no 'aws.s3' config is present, organization assets will be stored locally and this line is not needed.
# max_size_in_bytes: 1048576 # Optional, default is 1 MB
# location: 'organization_assets' # Optional subdirectory for local assets, default is 'organization_assets'
app_assets:
asset_host: "//cartodb-libs.global.ssl.fastly.net/cartodbui"
asset_host: "//cartodb.localhost"
avatars:
gravatar_enabled: true
base_url: '//cartodb-libs.global.ssl.fastly.net/cartodbui/assets/unversioned/images/avatars'
kinds: ['ghost', 'marker', 'mountain', 'pacman', 'planet', 'star']
base_url: '/assets/unversioned/images/avatars'
kinds: ['ghost', 'heart', 'marker', 'mountain', 'pacman', 'planet', 'star']
colors: ['green', 'orange', 'red', 'yellow']
dropbox_api_key: ""
gdrive:
@ -280,7 +299,7 @@ defaults: &defaults
# Use your Olark api id to enable it. If you remove this entry or don't define an app key, it won't be activated.
olark:
app_id: ''
enforce_non_empty_layer_css: true
enforce_non_empty_layer_css: false
users_dumps:
service:
port: 00000
@ -347,36 +366,10 @@ defaults: &defaults
ratelimit_concurrency: 3
ratelimit_ttl: 4
ratelimit_wait_secs: 0.1
datasources:
arcgis_enabled: false
salesforce_enabled: false
basemaps:
Nokia:
normal_day:
url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/normal.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
minZoom: '0'
maxZoom: '21'
className: "nokia_normal_day"
attribution: "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
name: 'Nokia normal Day'
terrain_day:
url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/terrain.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
minZoom: '0'
maxZoom: '21'
className: "nokia_terrain_day"
attribution: "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
name: 'Nokia Terrain Day'
satellite_day:
url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/satellite.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
minZoom: '0'
maxZoom: '21'
className: "nokia_satellite_day"
attribution: "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
name: 'Nokia Satellite Day'
hybrid_day:
url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/hybrid.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
minZoom: '0'
maxZoom: '21'
className: "nokia_hybrid_day"
attribution: "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
name: 'Nokia hybrid Day'
CARTO:
positron_rainbow_labels:
default: true
@ -526,6 +519,30 @@ defaults: &defaults
min_rows: 2000000
statement_timeout: 1800000
tolerance_px: 1.0
connectors:
odbc:
enabled: false
max_rows: nil
mysql:
enabled: true
max_rows: 500000
postgresql:
enabled: true
max_rows: 500000
hive:
enabled: false
max_rows: 500000
sqlserver:
enabled: false
max_rows: 500000
dataservices:
enabled:
geocoder_internal: true
hires_geocoder: false
isolines: false
routing: false
data_observatory: true
development:
<<: *defaults
@ -538,7 +555,6 @@ development:
url_purge_command: 'url.purge'
retries: 5
timeout: 5
enforce_non_empty_layer_css: false
test:
<<: *defaults

View File

@ -29,6 +29,10 @@ server {
proxy_pass http://127.0.0.1:8080;
}
location ^~ /assets {
root /cartodb/public;
}
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;

4
config/varnish.vcl Normal file
View File

@ -0,0 +1,4 @@
backend default {
.host = "127.0.0.1";
.port = "8080";
}

View File

@ -1,10 +1,9 @@
#!/bin/bash
PORT=3000
service postgresql start
service redis-server start
service varnish start
/opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl
service nginx start
cd /Windshaft-cartodb
node app.js development &
@ -16,5 +15,4 @@ cd /cartodb
source /usr/local/rvm/scripts/rvm
bundle exec script/restore_redis
bundle exec script/resque > resque.log 2>&1 &
bundle exec rails s -p $PORT
bundle exec thin start --threaded -p 3000 --threadpool-size 5

View File

@ -3,7 +3,7 @@
# Init script for template postgis
#
POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.1.2;
POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.2;
createdb -E UTF8 template_postgis;
createlang -d template_postgis plpgsql;
psql -d postgres -c "UPDATE pg_database SET datistemplate='true' \
@ -13,4 +13,3 @@ psql -d template_postgis -c "CREATE EXTENSION postgis_topology;"
psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;"
psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"
psql -c "CREATE EXTENSION plpythonu;"
psql -c "CREATE EXTENSION schema_triggers;"