* Embed nginx inside Docker image so all services are reversed proxied behind port 80

* Use Varnish 3 from source tarball, newer and OS Varnish could not disable telnet authentication required by Carto
* Remove sources of varnish, gdal, /cartodb/.git when no longer needed in order to make image smaller
* Dropped Nokia and GMAP basemaps, they did not work
* Dropped schema trigger installation, it is no longer needed
* Configure Carto in Python and NodeJS
* Build assets
* Use self as asset_host, assets are hosted by nginx instead of rails

Dockerfile

@@ -38,6 +38,7 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
     libproj-dev \
     gdal-bin \
     libgdal1-dev \
+    libgdal-dev \
     postgresql-9.5 \
     postgresql-client-9.5 \
     postgresql-contrib-9.5 \
@@ -47,11 +48,11 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
     postgresql-9.5-postgis-2.2 \
     postgresql-9.5-postgis-scripts \
     postgis \
+    liblwgeom-2.2-5 \
     ca-certificates \
     redis-server \
     python2.7-dev \
     python-setuptools \
-    varnish \
     imagemagick \
     libmapnik-dev \
     mapnik-utils \
@@ -59,6 +60,9 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
     python-argparse \
     python-gdal \
     python-chardet \
+    python-pip \
+    python-all-dev \
+    python-docutils \
     openssl \
     libreadline6 \
     zlib1g \
@@ -86,51 +90,64 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
     libgif-dev \
     libgmp-dev \
     libicu-dev \
+    wget \
+    nginx-light \
+    net-tools \
   --no-install-recommends && \
   rm -rf /var/lib/apt/lists/*
 
 RUN git config --global user.email you@example.com
 RUN git config --global user.name "Your Name"
 
+# Varnish 3, Ubuntu:16.04 comes with Varnish 4.1 which can't be run with anonymous admin telnet
+RUN cd /opt && \
+    wget https://repo.varnish-cache.org/source/varnish-3.0.7.tar.gz && \
+    tar -zxf varnish-3.0.7.tar.gz && \
+    cd varnish-3.0.7 && \
+    ./configure --prefix=/opt/varnish && \
+    make && \
+    make install &&
+    cd /opt &&
+    rm -rf varnish-3.0.7 varnish-3.0.7.tar.gz
+
 # ogr2ogr2 static build, see https://github.com/CartoDB/cartodb/wiki/How-to-build-gdal-and-ogr2ogr2
 # using cartodb instruction got error https://trac.osgeo.org/gdal/ticket/6073
 # https://github.com/OSGeo/gdal/compare/trunk...CartoDB:ogr2ogr2 has no code changes, so just use latest gdal tarball
 RUN cd /opt && \
-curl http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz -o gdal-2.1.1.tar.gz && \
+    curl http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz -o gdal-2.1.1.tar.gz && \
-tar -zxf gdal-2.1.1.tar.gz && \
+    tar -zxf gdal-2.1.1.tar.gz && \
-cd gdal-2.1.1 && \
+    cd gdal-2.1.1 && \
-./configure --disable-shared && \
+    ./configure --disable-shared && \
-make -j 4 && \
+    make -j 4 && \
-cp apps/ogr2ogr /usr/bin/ogr2ogr2 && \
+    cp apps/ogr2ogr /usr/bin/ogr2ogr2 && \
-rm -rf /opt/ogr2ogr2 /root/.gitconfig
+    cd /opt && \
+    rm -rf /opt/ogr2ogr2 /opt/gdal-2.1.1.tar.gz /root/.gitconfig /opt/gdal-2.1.1
 
 # Install NodeJS
-RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr
+RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr && \
+  npm install -g grunt-cli && \
+  npm install -g npm@~2.14.0
 
 # Install rvm
-RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 && \
-RUN curl -L https://get.rvm.io | bash -s stable --ruby
+    curl -L https://get.rvm.io | bash -s stable --ruby && \
-RUN echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc
+    echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc && \
-RUN /bin/bash -l -c rvm requirements
+    /bin/bash -l -c rvm requirements
-ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \
-RUN echo rvm_max_time_flag=15 >> ~/.rvmrc
+RUN echo rvm_max_time_flag=15 >> ~/.rvmrc && \
-RUN /bin/bash -l -c 'rvm install 2.2.3'
+    /bin/bash -l -c 'rvm install 2.2.3' && \
-RUN /bin/bash -l -c 'rvm use 2.2.3 --default'
+    /bin/bash -l -c 'rvm use 2.2.3 --default' && \
-RUN /bin/bash -l -c 'gem install bundle archive-tar-minitar'
+    /bin/bash -l -c 'gem install bundle archive-tar-minitar' && \
-
+    /bin/bash -l -c 'gem install bundler compass --no-doc --no-ri'
-# Install bundler
-RUN /bin/bash -l -c 'gem install bundler --no-doc --no-ri'
 
 # Setting PostgreSQL
-RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf
+RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf && \
+    service postgresql start && \
+    createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && \
+    createuser tileuser --no-createrole --no-createdb --no-superuser -U postgres && \
+    service postgresql stop
 
-# Install schema_triggers
+# Initialize template postgis db
-RUN git clone https://github.com/CartoDB/pg_schema_triggers.git && \
-      cd pg_schema_triggers && \
-      make all install && \
-      sed -i \
-      "/#shared_preload/a shared_preload_libraries = 'schema_triggers.so'" \
-      /etc/postgresql/9.5/main/postgresql.conf
 ADD ./template_postgis.sh /tmp/template_postgis.sh
 RUN service postgresql start && /bin/su postgres -c \
       /tmp/template_postgis.sh && service postgresql stop
@@ -138,6 +155,7 @@ RUN service postgresql start && /bin/su postgres -c \
 # Install cartodb extension
 RUN git clone https://github.com/CartoDB/cartodb-postgresql && \
       cd cartodb-postgresql && \
+      git checkout master && \
       PGUSER=postgres make install
 ADD ./cartodb_pgsql.sh /tmp/cartodb_pgsql.sh
 RUN service postgresql start && /bin/su postgres -c \
@@ -145,23 +163,22 @@ RUN service postgresql start && /bin/su postgres -c \
 
 # Install CartoDB API
 RUN git clone git://github.com/CartoDB/CartoDB-SQL-API.git && \
-      cd CartoDB-SQL-API && ./configure && npm install
+      cd CartoDB-SQL-API && npm install
 
 # Install Windshaft
 RUN git clone git://github.com/CartoDB/Windshaft-cartodb.git && \
-      cd Windshaft-cartodb && ./configure && npm install && mkdir logs
+      cd Windshaft-cartodb && git checkout master && npm install && mkdir logs
 
-# Install CartoDB (with the bug correction on bundle install)
+# Install CartoDB
-RUN git clone git://github.com/CartoDB/cartodb.git && \
+RUN git clone --recursive git://github.com/CartoDB/cartodb.git && \
-      cd cartodb && \
+    cd cartodb && \
-      perl -pi -e 's/jwt \(1\.5\.3\)/jwt (1.5.4)/' Gemfile.lock && \
+    git checkout master && \
-      /bin/bash -l -c 'bundle install' || \
+    npm install && \
-      /bin/bash -l -c "cd $(/bin/bash -l -c 'gem contents \
+    perl -pi -e 's/gdal==1\.10\.0/gdal==1.11.3/' python_requirements.txt && \
-            debugger-ruby_core_source' | grep CHANGELOG | sed -e \
+    pip install --no-use-wheel -r python_requirements.txt && \
-            's,CHANGELOG.md,,') && /bin/bash -l -c 'rake add_source \
+    /bin/bash -l -c 'bundle install' && \
-            VERSION=$(/bin/bash -l -c 'ruby --version' | awk \
+    /bin/bash -l -c 'bundle exec grunt --environment development' && \
-            '{print $2}' | sed -e 's,p55,-p55,' )' && cd /cartodb && \
+    rm -rf .git
-            /bin/bash -l -c 'bundle install'"
 
 # Geocoder SQL client + server
 RUN git clone https://github.com/CartoDB/data-services && \
@@ -186,18 +203,19 @@ ADD ./config/app_config.yml /cartodb/config/app_config.yml
 ADD ./config/database.yml /cartodb/config/database.yml
 ADD ./create_dev_user /cartodb/script/create_dev_user
 ADD ./setup_organization.sh /cartodb/script/setup_organization.sh
+ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default
+ADD ./config/varnish.vcl /etc/varnish.vcl
 ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications
+RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications && \
-RUN service postgresql start && service redis-server start && \
+    /opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl && \
-	bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && bash script/setup_organization.sh" && \
+    service postgresql start && service redis-server start && \
-# Enable CARTO Builder
+	bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && \
-#    bundle exec rake cartodb:features:enable_feature_for_all_users['editor-3'] && \
+    bash script/setup_organization.sh" && \
-#    bundle exec rake cartodb:features:enable_feature_for_all_users['explore_site']" && \
 	service postgresql stop && service redis-server stop
 
-EXPOSE 3000 8080 8181
+EXPOSE 80
 
-ENV GDAL_DATA /usr/share/gdal/1.10
+ENV GDAL_DATA /usr/share/gdal/1.11
 
 ADD ./startup.sh /opt/startup.sh
 

README.md

@@ -20,22 +20,17 @@ How to build the container:
 
 ```
 git clone https://github.com/sverhoeven/docker-cartodb.git
-docker build -t="sverhoeven/cartodb" docker-cartodb/
+docker build -t=sverhoeven/cartodb docker-cartodb/
 ```
 
 How to run the container:
 -------------------------
 
 ```
-docker run -d -p 3000:3000 -p 8080:8080 -p 8181:8181 sverhoeven/cartodb
+docker run -d -p 80:80 sverhoeven/cartodb
 ```
 
-The ports the cartodb container publishes must be combined behind a single [NGINX](http://nginx.org/) web server. 
+The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` as the machine where the web server is running.
-In the GitHub repo there is an [example NGINX config file (config/cartodb.nginx.proxy.conf)](https://github.com/sverhoeven/docker-cartodb/blob/master/config/cartodb.nginx.proxy.conf), which needs to be added to /etc/nginx/conf.d/ directory, after which the web server must be restarted.
-This will setup a reverse proxy for the CartoDB/imports (3000), SQL Api (8080) and Map api (8181) to default http port (80).
-Alternativly use instructions at https://hub.docker.com/r/spawnthink/cartodb-nginx/ to run NGINX as a docker container with the correct config file already in it.
-
-The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` to the machine where the web server is running.
 This can be done by adding cartodb.localhost alias to your hosts file. For example
 ```
 sudo sh -c 'echo 127.0.1.1 cartodb.localhost >> /etc/hosts'

config/app_config.yml

@@ -68,6 +68,8 @@ defaults: &defaults
     app_keys:
       editor: ''
       embeds: ''
+  #fullstory:
+    #org: 'XXXXX'
   facebook:
     app_id: ''
     admins: ''
@@ -102,6 +104,15 @@ defaults: &defaults
         wizard: ''
         filter: ''
         query: ''
+        applied_sql: ''
+        applied_cartocss: ''
+        modified_style_form: ''
+        completed_connection: ''
+        failed_connection: ''
+        created_analysis: ''
+        modified_analysis: ''
+        published_map: ''
+        exported_map: ''
       mailing_track:
         like_map: ''
         trending_map: ''
@@ -193,6 +204,7 @@ defaults: &defaults
       bucket_name: ''
       url_ttl: 7200
       async_long_uploads: false
+      s3_endpoint: ''
     uploads_path: 'public/uploads' # including 'uploads' assumes public path. Absolute path example: /tmp/exports/downloads
   importer:
     blacklisted_ip_addr: []
@@ -209,6 +221,7 @@ defaults: &defaults
       async_long_uploads: false
       proxy_uri:
       use_ssl: True
+      s3_endpoint: ''
 
     unp_temporal_folder: '/tmp/imports/'
     # It must end in `/uploads` and be accessible via HTTP, if relative will default to Rails.Root/#{uploads_path}
@@ -265,12 +278,18 @@ defaults: &defaults
   assets:
     s3_bucket_name: "tests"
     max_file_size: 5242880 # 5.megabytes
+    # Example for configuring organization assets.
+    # If 'aws.s3' configuration exists, S3 is used for storing assets and 'assets.organization.bucket' value is required. If 'aws.s3' config is not present, assets will be stored locally.
+    # organization:
+    #   bucket: "tests" # Required if 'aws.s3' config is present. Bucket must exist beforehand. If no 'aws.s3' config is present, organization assets will be stored locally and this line is not needed.
+    #   max_size_in_bytes: 1048576 # Optional, default is 1 MB
+    #   location: 'organization_assets' # Optional subdirectory for local assets, default is 'organization_assets'
   app_assets:
-    asset_host: "//cartodb-libs.global.ssl.fastly.net/cartodbui"
+    asset_host: "//cartodb.localhost"
   avatars:
     gravatar_enabled: true
-    base_url: '//cartodb-libs.global.ssl.fastly.net/cartodbui/assets/unversioned/images/avatars'
+    base_url: '/assets/unversioned/images/avatars'
-    kinds: ['ghost', 'marker', 'mountain', 'pacman', 'planet', 'star']
+    kinds: ['ghost', 'heart', 'marker', 'mountain', 'pacman', 'planet', 'star']
     colors: ['green', 'orange', 'red', 'yellow']
   dropbox_api_key: ""
   gdrive:
@@ -280,7 +299,7 @@ defaults: &defaults
   # Use your Olark api id to enable it. If you remove this entry or don't define an app key, it won't be activated.
   olark:
     app_id: ''
-  enforce_non_empty_layer_css: true
+  enforce_non_empty_layer_css: false
   users_dumps:
     service:
       port: 00000
@@ -347,36 +366,10 @@ defaults: &defaults
             ratelimit_concurrency:  3
             ratelimit_ttl:          4
             ratelimit_wait_secs:    0.1
+  datasources:
+    arcgis_enabled: false
+    salesforce_enabled: false
   basemaps:
-    Nokia:
-      normal_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/normal.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_normal_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia normal Day'
-      terrain_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/terrain.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_terrain_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia Terrain Day'
-      satellite_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/satellite.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_satellite_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia Satellite Day'
-      hybrid_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/hybrid.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_hybrid_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia hybrid Day'
     CARTO:
       positron_rainbow_labels:
         default: true
@@ -526,6 +519,30 @@ defaults: &defaults
     min_rows: 2000000
     statement_timeout: 1800000
     tolerance_px: 1.0
+  connectors:
+    odbc:
+      enabled: false
+      max_rows: nil
+    mysql:
+      enabled: true
+      max_rows: 500000
+    postgresql:
+      enabled: true
+      max_rows: 500000
+    hive:
+      enabled: false
+      max_rows: 500000
+    sqlserver:
+      enabled: false
+      max_rows: 500000
+  dataservices:
+    enabled:
+        geocoder_internal: true
+        hires_geocoder: false
+        isolines: false
+        routing: false
+        data_observatory: true
+
 
 development:
   <<: *defaults
@@ -538,7 +555,6 @@ development:
     url_purge_command: 'url.purge'
     retries: 5
     timeout: 5
-  enforce_non_empty_layer_css: false
 
 test:
   <<: *defaults

config/cartodb.nginx.proxy.conf

@@ -29,6 +29,10 @@ server {
     proxy_pass http://127.0.0.1:8080;
   }
 
+  location ^~ /assets {
+    root /cartodb/public;
+  }
+
   location / {
     proxy_set_header        Host $host;
     proxy_set_header        X-Real-IP $remote_addr;

config/varnish.vcl

@@ -0,0 +1,4 @@
+backend default {
+  .host = "127.0.0.1";
+  .port = "8080";
+}

startup.sh

@@ -1,10 +1,9 @@
 #!/bin/bash
 
-PORT=3000
-
 service postgresql start
 service redis-server start
-service varnish start
+/opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl
+service nginx start
 
 cd /Windshaft-cartodb
 node app.js development &
@@ -16,5 +15,4 @@ cd /cartodb
 source /usr/local/rvm/scripts/rvm
 bundle exec script/restore_redis
 bundle exec script/resque > resque.log 2>&1 &
-bundle exec rails s -p $PORT
+bundle exec thin start --threaded -p 3000 --threadpool-size 5
-

template_postgis.sh

@@ -3,7 +3,7 @@
 # Init script for template postgis
 #
 
-POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.1.2;
+POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.2;
 createdb -E UTF8 template_postgis;
 createlang -d template_postgis plpgsql;
 psql -d postgres -c "UPDATE pg_database SET datistemplate='true' \
@@ -13,4 +13,3 @@ psql -d template_postgis -c "CREATE EXTENSION postgis_topology;"
 psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;"
 psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"
 psql -c "CREATE EXTENSION plpythonu;"
-psql -c "CREATE EXTENSION schema_triggers;"