* Embed nginx inside Docker image so all services are reversed proxied behind port 80

* Use Varnish 3 from source tarball, newer and OS Varnish could not disable telnet authentication required by Carto * Remove sources of varnish, gdal, /cartodb/.git when no longer needed in order to make image smaller * Dropped Nokia and GMAP basemaps, they did not work * Dropped schema trigger installation, it is no longer needed * Configure Carto in Python and NodeJS * Build assets * Use self as asset_host, assets are hosted by nginx instead of rails
2017-04-03 22:46:52 +02:00 · 2017-04-03 22:46:52 +02:00 · bbf2c54325
commit bbf2c54325
parent 3397aed11d
7 changed files with 133 additions and 99 deletions
--- a/104
+++ b/104
@ -38,6 +38,7 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
    libproj-dev \
    gdal-bin \
    libgdal1-dev \
+    libgdal-dev \
    postgresql-9.5 \
    postgresql-client-9.5 \
    postgresql-contrib-9.5 \
@ -47,11 +48,11 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
    postgresql-9.5-postgis-2.2 \
    postgresql-9.5-postgis-scripts \
    postgis \
+    liblwgeom-2.2-5 \
    ca-certificates \
    redis-server \
    python2.7-dev \
    python-setuptools \
-    varnish \
    imagemagick \
    libmapnik-dev \
    mapnik-utils \
@ -59,6 +60,9 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
    python-argparse \
    python-gdal \
    python-chardet \
+    python-pip \
+    python-all-dev \
+    python-docutils \
    openssl \
    libreadline6 \
    zlib1g \
@ -86,12 +90,26 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \
    libgif-dev \
    libgmp-dev \
    libicu-dev \
+    wget \
+    nginx-light \
+    net-tools \
  --no-install-recommends && \
  rm -rf /var/lib/apt/lists/*

 RUN git config --global user.email you@example.com
 RUN git config --global user.name "Your Name"

+# Varnish 3, Ubuntu:16.04 comes with Varnish 4.1 which can't be run with anonymous admin telnet
+RUN cd /opt && \
+    wget https://repo.varnish-cache.org/source/varnish-3.0.7.tar.gz && \
+    tar -zxf varnish-3.0.7.tar.gz && \
+    cd varnish-3.0.7 && \
+    ./configure --prefix=/opt/varnish && \
+    make && \
+    make install &&
+    cd /opt &&
+    rm -rf varnish-3.0.7 varnish-3.0.7.tar.gz
+
 # ogr2ogr2 static build, see https://github.com/CartoDB/cartodb/wiki/How-to-build-gdal-and-ogr2ogr2
 # using cartodb instruction got error https://trac.osgeo.org/gdal/ticket/6073
 # https://github.com/OSGeo/gdal/compare/trunk...CartoDB:ogr2ogr2 has no code changes, so just use latest gdal tarball
@ -102,35 +120,34 @@ cd gdal-2.1.1 && \
    ./configure --disable-shared && \
    make -j 4 && \
    cp apps/ogr2ogr /usr/bin/ogr2ogr2 && \
-rm -rf /opt/ogr2ogr2 /root/.gitconfig
+    cd /opt && \
+    rm -rf /opt/ogr2ogr2 /opt/gdal-2.1.1.tar.gz /root/.gitconfig /opt/gdal-2.1.1

 # Install NodeJS
-RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr
+RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr && \
+  npm install -g grunt-cli && \
+  npm install -g npm@~2.14.0

 # Install rvm
-RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3
-RUN curl -L https://get.rvm.io | bash -s stable --ruby
-RUN echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc
-RUN /bin/bash -l -c rvm requirements
-ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-RUN echo rvm_max_time_flag=15 >> ~/.rvmrc
-RUN /bin/bash -l -c 'rvm install 2.2.3'
-RUN /bin/bash -l -c 'rvm use 2.2.3 --default'
-RUN /bin/bash -l -c 'gem install bundle archive-tar-minitar'
-
-# Install bundler
-RUN /bin/bash -l -c 'gem install bundler --no-doc --no-ri'
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 && \
+    curl -L https://get.rvm.io | bash -s stable --ruby && \
+    echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc && \
+    /bin/bash -l -c rvm requirements
+ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \
+RUN echo rvm_max_time_flag=15 >> ~/.rvmrc && \
+    /bin/bash -l -c 'rvm install 2.2.3' && \
+    /bin/bash -l -c 'rvm use 2.2.3 --default' && \
+    /bin/bash -l -c 'gem install bundle archive-tar-minitar' && \
+    /bin/bash -l -c 'gem install bundler compass --no-doc --no-ri'

 # Setting PostgreSQL
-RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf
+RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf && \
+    service postgresql start && \
+    createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && \
+    createuser tileuser --no-createrole --no-createdb --no-superuser -U postgres && \
+    service postgresql stop

-# Install schema_triggers
-RUN git clone https://github.com/CartoDB/pg_schema_triggers.git && \
-      cd pg_schema_triggers && \
-      make all install && \
-      sed -i \
-      "/#shared_preload/a shared_preload_libraries = 'schema_triggers.so'" \
-      /etc/postgresql/9.5/main/postgresql.conf
+# Initialize template postgis db
 ADD ./template_postgis.sh /tmp/template_postgis.sh
 RUN service postgresql start && /bin/su postgres -c \
      /tmp/template_postgis.sh && service postgresql stop
@ -138,6 +155,7 @@ RUN service postgresql start && /bin/su postgres -c \
 # Install cartodb extension
 RUN git clone https://github.com/CartoDB/cartodb-postgresql && \
      cd cartodb-postgresql && \
+      git checkout master && \
      PGUSER=postgres make install
 ADD ./cartodb_pgsql.sh /tmp/cartodb_pgsql.sh
 RUN service postgresql start && /bin/su postgres -c \
@ -145,23 +163,22 @@ RUN service postgresql start && /bin/su postgres -c \

 # Install CartoDB API
 RUN git clone git://github.com/CartoDB/CartoDB-SQL-API.git && \
-      cd CartoDB-SQL-API && ./configure && npm install
+      cd CartoDB-SQL-API && npm install

 # Install Windshaft
 RUN git clone git://github.com/CartoDB/Windshaft-cartodb.git && \
-      cd Windshaft-cartodb && ./configure && npm install && mkdir logs
+      cd Windshaft-cartodb && git checkout master && npm install && mkdir logs

-# Install CartoDB (with the bug correction on bundle install)
-RUN git clone git://github.com/CartoDB/cartodb.git && \
+# Install CartoDB
+RUN git clone --recursive git://github.com/CartoDB/cartodb.git && \
    cd cartodb && \
-      perl -pi -e 's/jwt \(1\.5\.3\)/jwt (1.5.4)/' Gemfile.lock && \
-      /bin/bash -l -c 'bundle install' || \
-      /bin/bash -l -c "cd $(/bin/bash -l -c 'gem contents \
-            debugger-ruby_core_source' | grep CHANGELOG | sed -e \
-            's,CHANGELOG.md,,') && /bin/bash -l -c 'rake add_source \
-            VERSION=$(/bin/bash -l -c 'ruby --version' | awk \
-            '{print $2}' | sed -e 's,p55,-p55,' )' && cd /cartodb && \
-            /bin/bash -l -c 'bundle install'"
+    git checkout master && \
+    npm install && \
+    perl -pi -e 's/gdal==1\.10\.0/gdal==1.11.3/' python_requirements.txt && \
+    pip install --no-use-wheel -r python_requirements.txt && \
+    /bin/bash -l -c 'bundle install' && \
+    /bin/bash -l -c 'bundle exec grunt --environment development' && \
+    rm -rf .git

 # Geocoder SQL client + server
 RUN git clone https://github.com/CartoDB/data-services && \
@ -186,18 +203,19 @@ ADD ./config/app_config.yml /cartodb/config/app_config.yml
 ADD ./config/database.yml /cartodb/config/database.yml
 ADD ./create_dev_user /cartodb/script/create_dev_user
 ADD ./setup_organization.sh /cartodb/script/setup_organization.sh
+ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default
+ADD ./config/varnish.vcl /etc/varnish.vcl
 ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications
-RUN service postgresql start && service redis-server start && \
-	bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && bash script/setup_organization.sh" && \
-# Enable CARTO Builder
-#    bundle exec rake cartodb:features:enable_feature_for_all_users['editor-3'] && \
-#    bundle exec rake cartodb:features:enable_feature_for_all_users['explore_site']" && \
+RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications && \
+    /opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl && \
+    service postgresql start && service redis-server start && \
+	bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && \
+    bash script/setup_organization.sh" && \
 	service postgresql stop && service redis-server stop

-EXPOSE 3000 8080 8181
+EXPOSE 80

-ENV GDAL_DATA /usr/share/gdal/1.10
+ENV GDAL_DATA /usr/share/gdal/1.11

 ADD ./startup.sh /opt/startup.sh

--- a/README.md
+++ b/README.md
@ -20,22 +20,17 @@ How to build the container:

 ```
 git clone https://github.com/sverhoeven/docker-cartodb.git
-docker build -t="sverhoeven/cartodb" docker-cartodb/
+docker build -t=sverhoeven/cartodb docker-cartodb/
 ```

 How to run the container:
 -------------------------

 ```
-docker run -d -p 3000:3000 -p 8080:8080 -p 8181:8181 sverhoeven/cartodb
+docker run -d -p 80:80 sverhoeven/cartodb
 ```

-The ports the cartodb container publishes must be combined behind a single [NGINX](http://nginx.org/) web server. 
-In the GitHub repo there is an [example NGINX config file (config/cartodb.nginx.proxy.conf)](https://github.com/sverhoeven/docker-cartodb/blob/master/config/cartodb.nginx.proxy.conf), which needs to be added to /etc/nginx/conf.d/ directory, after which the web server must be restarted.
-This will setup a reverse proxy for the CartoDB/imports (3000), SQL Api (8080) and Map api (8181) to default http port (80).
-Alternativly use instructions at https://hub.docker.com/r/spawnthink/cartodb-nginx/ to run NGINX as a docker container with the correct config file already in it.
-
-The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` to the machine where the web server is running.
+The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` as the machine where the web server is running.
 This can be done by adding cartodb.localhost alias to your hosts file. For example
 ```
 sudo sh -c 'echo 127.0.1.1 cartodb.localhost >> /etc/hosts'
--- a/config/app_config.yml
+++ b/config/app_config.yml
@ -68,6 +68,8 @@ defaults: &defaults
    app_keys:
      editor: ''
      embeds: ''
+  #fullstory:
+    #org: 'XXXXX'
  facebook:
    app_id: ''
    admins: ''
@ -102,6 +104,15 @@ defaults: &defaults
        wizard: ''
        filter: ''
        query: ''
+        applied_sql: ''
+        applied_cartocss: ''
+        modified_style_form: ''
+        completed_connection: ''
+        failed_connection: ''
+        created_analysis: ''
+        modified_analysis: ''
+        published_map: ''
+        exported_map: ''
      mailing_track:
        like_map: ''
        trending_map: ''
@ -193,6 +204,7 @@ defaults: &defaults
      bucket_name: ''
      url_ttl: 7200
      async_long_uploads: false
+      s3_endpoint: ''
    uploads_path: 'public/uploads' # including 'uploads' assumes public path. Absolute path example: /tmp/exports/downloads
  importer:
    blacklisted_ip_addr: []
@ -209,6 +221,7 @@ defaults: &defaults
      async_long_uploads: false
      proxy_uri:
      use_ssl: True
+      s3_endpoint: ''

    unp_temporal_folder: '/tmp/imports/'
    # It must end in `/uploads` and be accessible via HTTP, if relative will default to Rails.Root/#{uploads_path}
@ -265,12 +278,18 @@ defaults: &defaults
  assets:
    s3_bucket_name: "tests"
    max_file_size: 5242880 # 5.megabytes
+    # Example for configuring organization assets.
+    # If 'aws.s3' configuration exists, S3 is used for storing assets and 'assets.organization.bucket' value is required. If 'aws.s3' config is not present, assets will be stored locally.
+    # organization:
+    #   bucket: "tests" # Required if 'aws.s3' config is present. Bucket must exist beforehand. If no 'aws.s3' config is present, organization assets will be stored locally and this line is not needed.
+    #   max_size_in_bytes: 1048576 # Optional, default is 1 MB
+    #   location: 'organization_assets' # Optional subdirectory for local assets, default is 'organization_assets'
  app_assets:
-    asset_host: "//cartodb-libs.global.ssl.fastly.net/cartodbui"
+    asset_host: "//cartodb.localhost"
  avatars:
    gravatar_enabled: true
-    base_url: '//cartodb-libs.global.ssl.fastly.net/cartodbui/assets/unversioned/images/avatars'
-    kinds: ['ghost', 'marker', 'mountain', 'pacman', 'planet', 'star']
+    base_url: '/assets/unversioned/images/avatars'
+    kinds: ['ghost', 'heart', 'marker', 'mountain', 'pacman', 'planet', 'star']
    colors: ['green', 'orange', 'red', 'yellow']
  dropbox_api_key: ""
  gdrive:
@ -280,7 +299,7 @@ defaults: &defaults
  # Use your Olark api id to enable it. If you remove this entry or don't define an app key, it won't be activated.
  olark:
    app_id: ''
-  enforce_non_empty_layer_css: true
+  enforce_non_empty_layer_css: false
  users_dumps:
    service:
      port: 00000
@ -347,36 +366,10 @@ defaults: &defaults
            ratelimit_concurrency:  3
            ratelimit_ttl:          4
            ratelimit_wait_secs:    0.1
+  datasources:
+    arcgis_enabled: false
+    salesforce_enabled: false
  basemaps:
-    Nokia:
-      normal_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/normal.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_normal_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia normal Day'
-      terrain_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/terrain.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_terrain_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia Terrain Day'
-      satellite_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/satellite.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_satellite_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia Satellite Day'
-      hybrid_day:
-        url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/hybrid.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24'
-        minZoom: '0'
-        maxZoom: '21'
-        className:    "nokia_hybrid_day"
-        attribution:  "©2012 Nokia <a href='http://here.net/services/terms' target='_blank'>Terms of use</a>"
-        name: 'Nokia hybrid Day'
    CARTO:
      positron_rainbow_labels:
        default: true
@ -526,6 +519,30 @@ defaults: &defaults
    min_rows: 2000000
    statement_timeout: 1800000
    tolerance_px: 1.0
+  connectors:
+    odbc:
+      enabled: false
+      max_rows: nil
+    mysql:
+      enabled: true
+      max_rows: 500000
+    postgresql:
+      enabled: true
+      max_rows: 500000
+    hive:
+      enabled: false
+      max_rows: 500000
+    sqlserver:
+      enabled: false
+      max_rows: 500000
+  dataservices:
+    enabled:
+        geocoder_internal: true
+        hires_geocoder: false
+        isolines: false
+        routing: false
+        data_observatory: true
+

 development:
  <<: *defaults
@ -538,7 +555,6 @@ development:
    url_purge_command: 'url.purge'
    retries: 5
    timeout: 5
-  enforce_non_empty_layer_css: false

 test:
  <<: *defaults
--- a/config/cartodb.nginx.proxy.conf
+++ b/config/cartodb.nginx.proxy.conf
@ -29,6 +29,10 @@ server {
    proxy_pass http://127.0.0.1:8080;
  }

+  location ^~ /assets {
+    root /cartodb/public;
+  }
+
  location / {
    proxy_set_header        Host $host;
    proxy_set_header        X-Real-IP $remote_addr;
--- a/config/varnish.vcl
+++ b/config/varnish.vcl
@ -0,0 +1,4 @@
+backend default {
+  .host = "127.0.0.1";
+  .port = "8080";
+}
--- a/startup.sh
+++ b/startup.sh
@ -1,10 +1,9 @@
 #!/bin/bash

-PORT=3000
-
 service postgresql start
 service redis-server start
-service varnish start
+/opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl
+service nginx start

 cd /Windshaft-cartodb
 node app.js development &
@ -16,5 +15,4 @@ cd /cartodb
 source /usr/local/rvm/scripts/rvm
 bundle exec script/restore_redis
 bundle exec script/resque > resque.log 2>&1 &
-bundle exec rails s -p $PORT
-
+bundle exec thin start --threaded -p 3000 --threadpool-size 5
--- a/template_postgis.sh
+++ b/template_postgis.sh
@ -3,7 +3,7 @@
 # Init script for template postgis
 #

-POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.1.2;
+POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.2;
 createdb -E UTF8 template_postgis;
 createlang -d template_postgis plpgsql;
 psql -d postgres -c "UPDATE pg_database SET datistemplate='true' \
@ -13,4 +13,3 @@ psql -d template_postgis -c "CREATE EXTENSION postgis_topology;"
 psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;"
 psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"
 psql -c "CREATE EXTENSION plpythonu;"
-psql -c "CREATE EXTENSION schema_triggers;"