diff --git a/Dockerfile b/Dockerfile index 7848e3b..7d989fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \ libproj-dev \ gdal-bin \ libgdal1-dev \ + libgdal-dev \ postgresql-9.5 \ postgresql-client-9.5 \ postgresql-contrib-9.5 \ @@ -47,11 +48,11 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \ postgresql-9.5-postgis-2.2 \ postgresql-9.5-postgis-scripts \ postgis \ + liblwgeom-2.2-5 \ ca-certificates \ redis-server \ python2.7-dev \ python-setuptools \ - varnish \ imagemagick \ libmapnik-dev \ mapnik-utils \ @@ -59,6 +60,9 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \ python-argparse \ python-gdal \ python-chardet \ + python-pip \ + python-all-dev \ + python-docutils \ openssl \ libreadline6 \ zlib1g \ @@ -86,51 +90,64 @@ RUN useradd -m -d /home/cartodb -s /bin/bash cartodb && \ libgif-dev \ libgmp-dev \ libicu-dev \ + wget \ + nginx-light \ + net-tools \ --no-install-recommends && \ rm -rf /var/lib/apt/lists/* RUN git config --global user.email you@example.com RUN git config --global user.name "Your Name" +# Varnish 3, Ubuntu:16.04 comes with Varnish 4.1 which can't be run with anonymous admin telnet +RUN cd /opt && \ + wget https://repo.varnish-cache.org/source/varnish-3.0.7.tar.gz && \ + tar -zxf varnish-3.0.7.tar.gz && \ + cd varnish-3.0.7 && \ + ./configure --prefix=/opt/varnish && \ + make && \ + make install && + cd /opt && + rm -rf varnish-3.0.7 varnish-3.0.7.tar.gz + # ogr2ogr2 static build, see https://github.com/CartoDB/cartodb/wiki/How-to-build-gdal-and-ogr2ogr2 # using cartodb instruction got error https://trac.osgeo.org/gdal/ticket/6073 # https://github.com/OSGeo/gdal/compare/trunk...CartoDB:ogr2ogr2 has no code changes, so just use latest gdal tarball RUN cd /opt && \ -curl http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz -o gdal-2.1.1.tar.gz && \ -tar -zxf gdal-2.1.1.tar.gz && \ -cd gdal-2.1.1 && \ -./configure --disable-shared && \ -make -j 4 && \ -cp apps/ogr2ogr /usr/bin/ogr2ogr2 && \ -rm -rf /opt/ogr2ogr2 /root/.gitconfig + curl http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz -o gdal-2.1.1.tar.gz && \ + tar -zxf gdal-2.1.1.tar.gz && \ + cd gdal-2.1.1 && \ + ./configure --disable-shared && \ + make -j 4 && \ + cp apps/ogr2ogr /usr/bin/ogr2ogr2 && \ + cd /opt && \ + rm -rf /opt/ogr2ogr2 /opt/gdal-2.1.1.tar.gz /root/.gitconfig /opt/gdal-2.1.1 # Install NodeJS -RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr +RUN curl https://nodejs.org/download/release/v0.10.41/node-v0.10.41-linux-x64.tar.gz| tar -zxf - --strip-components=1 -C /usr && \ + npm install -g grunt-cli && \ + npm install -g npm@~2.14.0 # Install rvm -RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 -RUN curl -L https://get.rvm.io | bash -s stable --ruby -RUN echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc -RUN /bin/bash -l -c rvm requirements -ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -RUN echo rvm_max_time_flag=15 >> ~/.rvmrc -RUN /bin/bash -l -c 'rvm install 2.2.3' -RUN /bin/bash -l -c 'rvm use 2.2.3 --default' -RUN /bin/bash -l -c 'gem install bundle archive-tar-minitar' - -# Install bundler -RUN /bin/bash -l -c 'gem install bundler --no-doc --no-ri' +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 && \ + curl -L https://get.rvm.io | bash -s stable --ruby && \ + echo 'source /usr/local/rvm/scripts/rvm' >> /etc/bash.bashrc && \ + /bin/bash -l -c rvm requirements +ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \ +RUN echo rvm_max_time_flag=15 >> ~/.rvmrc && \ + /bin/bash -l -c 'rvm install 2.2.3' && \ + /bin/bash -l -c 'rvm use 2.2.3 --default' && \ + /bin/bash -l -c 'gem install bundle archive-tar-minitar' && \ + /bin/bash -l -c 'gem install bundler compass --no-doc --no-ri' # Setting PostgreSQL -RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf - -# Install schema_triggers -RUN git clone https://github.com/CartoDB/pg_schema_triggers.git && \ - cd pg_schema_triggers && \ - make all install && \ - sed -i \ - "/#shared_preload/a shared_preload_libraries = 'schema_triggers.so'" \ - /etc/postgresql/9.5/main/postgresql.conf +RUN sed -i 's/\(peer\|md5\)/trust/' /etc/postgresql/9.5/main/pg_hba.conf && \ + service postgresql start && \ + createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && \ + createuser tileuser --no-createrole --no-createdb --no-superuser -U postgres && \ + service postgresql stop + +# Initialize template postgis db ADD ./template_postgis.sh /tmp/template_postgis.sh RUN service postgresql start && /bin/su postgres -c \ /tmp/template_postgis.sh && service postgresql stop @@ -138,6 +155,7 @@ RUN service postgresql start && /bin/su postgres -c \ # Install cartodb extension RUN git clone https://github.com/CartoDB/cartodb-postgresql && \ cd cartodb-postgresql && \ + git checkout master && \ PGUSER=postgres make install ADD ./cartodb_pgsql.sh /tmp/cartodb_pgsql.sh RUN service postgresql start && /bin/su postgres -c \ @@ -145,23 +163,22 @@ RUN service postgresql start && /bin/su postgres -c \ # Install CartoDB API RUN git clone git://github.com/CartoDB/CartoDB-SQL-API.git && \ - cd CartoDB-SQL-API && ./configure && npm install + cd CartoDB-SQL-API && npm install # Install Windshaft RUN git clone git://github.com/CartoDB/Windshaft-cartodb.git && \ - cd Windshaft-cartodb && ./configure && npm install && mkdir logs - -# Install CartoDB (with the bug correction on bundle install) -RUN git clone git://github.com/CartoDB/cartodb.git && \ - cd cartodb && \ - perl -pi -e 's/jwt \(1\.5\.3\)/jwt (1.5.4)/' Gemfile.lock && \ - /bin/bash -l -c 'bundle install' || \ - /bin/bash -l -c "cd $(/bin/bash -l -c 'gem contents \ - debugger-ruby_core_source' | grep CHANGELOG | sed -e \ - 's,CHANGELOG.md,,') && /bin/bash -l -c 'rake add_source \ - VERSION=$(/bin/bash -l -c 'ruby --version' | awk \ - '{print $2}' | sed -e 's,p55,-p55,' )' && cd /cartodb && \ - /bin/bash -l -c 'bundle install'" + cd Windshaft-cartodb && git checkout master && npm install && mkdir logs + +# Install CartoDB +RUN git clone --recursive git://github.com/CartoDB/cartodb.git && \ + cd cartodb && \ + git checkout master && \ + npm install && \ + perl -pi -e 's/gdal==1\.10\.0/gdal==1.11.3/' python_requirements.txt && \ + pip install --no-use-wheel -r python_requirements.txt && \ + /bin/bash -l -c 'bundle install' && \ + /bin/bash -l -c 'bundle exec grunt --environment development' && \ + rm -rf .git # Geocoder SQL client + server RUN git clone https://github.com/CartoDB/data-services && \ @@ -186,18 +203,19 @@ ADD ./config/app_config.yml /cartodb/config/app_config.yml ADD ./config/database.yml /cartodb/config/database.yml ADD ./create_dev_user /cartodb/script/create_dev_user ADD ./setup_organization.sh /cartodb/script/setup_organization.sh +ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default +ADD ./config/varnish.vcl /etc/varnish.vcl ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications -RUN service postgresql start && service redis-server start && \ - bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && bash script/setup_organization.sh" && \ -# Enable CARTO Builder -# bundle exec rake cartodb:features:enable_feature_for_all_users['editor-3'] && \ -# bundle exec rake cartodb:features:enable_feature_for_all_users['explore_site']" && \ +RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications && \ + /opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl && \ + service postgresql start && service redis-server start && \ + bash -l -c "cd /cartodb && bash script/create_dev_user || bash script/create_dev_user && \ + bash script/setup_organization.sh" && \ service postgresql stop && service redis-server stop -EXPOSE 3000 8080 8181 +EXPOSE 80 -ENV GDAL_DATA /usr/share/gdal/1.10 +ENV GDAL_DATA /usr/share/gdal/1.11 ADD ./startup.sh /opt/startup.sh diff --git a/README.md b/README.md index 5af098d..9c7f78f 100644 --- a/README.md +++ b/README.md @@ -20,22 +20,17 @@ How to build the container: ``` git clone https://github.com/sverhoeven/docker-cartodb.git -docker build -t="sverhoeven/cartodb" docker-cartodb/ +docker build -t=sverhoeven/cartodb docker-cartodb/ ``` How to run the container: ------------------------- ``` -docker run -d -p 3000:3000 -p 8080:8080 -p 8181:8181 sverhoeven/cartodb +docker run -d -p 80:80 sverhoeven/cartodb ``` -The ports the cartodb container publishes must be combined behind a single [NGINX](http://nginx.org/) web server. -In the GitHub repo there is an [example NGINX config file (config/cartodb.nginx.proxy.conf)](https://github.com/sverhoeven/docker-cartodb/blob/master/config/cartodb.nginx.proxy.conf), which needs to be added to /etc/nginx/conf.d/ directory, after which the web server must be restarted. -This will setup a reverse proxy for the CartoDB/imports (3000), SQL Api (8080) and Map api (8181) to default http port (80). -Alternativly use instructions at https://hub.docker.com/r/spawnthink/cartodb-nginx/ to run NGINX as a docker container with the correct config file already in it. - -The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` to the machine where the web server is running. +The CartoDB instance has been configured with the hostname `cartodb.localhost`, this means the web browser and web server need to be able to resolve `cartodb.localhost` as the machine where the web server is running. This can be done by adding cartodb.localhost alias to your hosts file. For example ``` sudo sh -c 'echo 127.0.1.1 cartodb.localhost >> /etc/hosts' diff --git a/config/app_config.yml b/config/app_config.yml index 9b1965e..151b1fc 100644 --- a/config/app_config.yml +++ b/config/app_config.yml @@ -68,6 +68,8 @@ defaults: &defaults app_keys: editor: '' embeds: '' + #fullstory: + #org: 'XXXXX' facebook: app_id: '' admins: '' @@ -102,6 +104,15 @@ defaults: &defaults wizard: '' filter: '' query: '' + applied_sql: '' + applied_cartocss: '' + modified_style_form: '' + completed_connection: '' + failed_connection: '' + created_analysis: '' + modified_analysis: '' + published_map: '' + exported_map: '' mailing_track: like_map: '' trending_map: '' @@ -193,6 +204,7 @@ defaults: &defaults bucket_name: '' url_ttl: 7200 async_long_uploads: false + s3_endpoint: '' uploads_path: 'public/uploads' # including 'uploads' assumes public path. Absolute path example: /tmp/exports/downloads importer: blacklisted_ip_addr: [] @@ -209,6 +221,7 @@ defaults: &defaults async_long_uploads: false proxy_uri: use_ssl: True + s3_endpoint: '' unp_temporal_folder: '/tmp/imports/' # It must end in `/uploads` and be accessible via HTTP, if relative will default to Rails.Root/#{uploads_path} @@ -265,12 +278,18 @@ defaults: &defaults assets: s3_bucket_name: "tests" max_file_size: 5242880 # 5.megabytes + # Example for configuring organization assets. + # If 'aws.s3' configuration exists, S3 is used for storing assets and 'assets.organization.bucket' value is required. If 'aws.s3' config is not present, assets will be stored locally. + # organization: + # bucket: "tests" # Required if 'aws.s3' config is present. Bucket must exist beforehand. If no 'aws.s3' config is present, organization assets will be stored locally and this line is not needed. + # max_size_in_bytes: 1048576 # Optional, default is 1 MB + # location: 'organization_assets' # Optional subdirectory for local assets, default is 'organization_assets' app_assets: - asset_host: "//cartodb-libs.global.ssl.fastly.net/cartodbui" + asset_host: "//cartodb.localhost" avatars: gravatar_enabled: true - base_url: '//cartodb-libs.global.ssl.fastly.net/cartodbui/assets/unversioned/images/avatars' - kinds: ['ghost', 'marker', 'mountain', 'pacman', 'planet', 'star'] + base_url: '/assets/unversioned/images/avatars' + kinds: ['ghost', 'heart', 'marker', 'mountain', 'pacman', 'planet', 'star'] colors: ['green', 'orange', 'red', 'yellow'] dropbox_api_key: "" gdrive: @@ -280,7 +299,7 @@ defaults: &defaults # Use your Olark api id to enable it. If you remove this entry or don't define an app key, it won't be activated. olark: app_id: '' - enforce_non_empty_layer_css: true + enforce_non_empty_layer_css: false users_dumps: service: port: 00000 @@ -347,36 +366,10 @@ defaults: &defaults ratelimit_concurrency: 3 ratelimit_ttl: 4 ratelimit_wait_secs: 0.1 + datasources: + arcgis_enabled: false + salesforce_enabled: false basemaps: - Nokia: - normal_day: - url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/normal.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24' - minZoom: '0' - maxZoom: '21' - className: "nokia_normal_day" - attribution: "©2012 Nokia Terms of use" - name: 'Nokia normal Day' - terrain_day: - url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/terrain.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24' - minZoom: '0' - maxZoom: '21' - className: "nokia_terrain_day" - attribution: "©2012 Nokia Terms of use" - name: 'Nokia Terrain Day' - satellite_day: - url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/satellite.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24' - minZoom: '0' - maxZoom: '21' - className: "nokia_satellite_day" - attribution: "©2012 Nokia Terms of use" - name: 'Nokia Satellite Day' - hybrid_day: - url: 'https://maps.nlp.nokia.com/maptiler/v2/maptile/newest/hybrid.day/{z}/{x}/{y}/256/png8?lg=eng&token=A7tBPacePg9Mj_zghvKt9Q&app_id=KuYppsdXZznpffJsKT24' - minZoom: '0' - maxZoom: '21' - className: "nokia_hybrid_day" - attribution: "©2012 Nokia Terms of use" - name: 'Nokia hybrid Day' CARTO: positron_rainbow_labels: default: true @@ -526,6 +519,30 @@ defaults: &defaults min_rows: 2000000 statement_timeout: 1800000 tolerance_px: 1.0 + connectors: + odbc: + enabled: false + max_rows: nil + mysql: + enabled: true + max_rows: 500000 + postgresql: + enabled: true + max_rows: 500000 + hive: + enabled: false + max_rows: 500000 + sqlserver: + enabled: false + max_rows: 500000 + dataservices: + enabled: + geocoder_internal: true + hires_geocoder: false + isolines: false + routing: false + data_observatory: true + development: <<: *defaults @@ -538,7 +555,6 @@ development: url_purge_command: 'url.purge' retries: 5 timeout: 5 - enforce_non_empty_layer_css: false test: <<: *defaults diff --git a/config/cartodb.nginx.proxy.conf b/config/cartodb.nginx.proxy.conf index 9d1c996..b51a24a 100644 --- a/config/cartodb.nginx.proxy.conf +++ b/config/cartodb.nginx.proxy.conf @@ -29,6 +29,10 @@ server { proxy_pass http://127.0.0.1:8080; } + location ^~ /assets { + root /cartodb/public; + } + location / { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/config/varnish.vcl b/config/varnish.vcl new file mode 100644 index 0000000..0e3fed4 --- /dev/null +++ b/config/varnish.vcl @@ -0,0 +1,4 @@ +backend default { + .host = "127.0.0.1"; + .port = "8080"; +} diff --git a/startup.sh b/startup.sh index 1a05ec5..490e6de 100755 --- a/startup.sh +++ b/startup.sh @@ -1,10 +1,9 @@ #!/bin/bash -PORT=3000 - service postgresql start service redis-server start -service varnish start +/opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl +service nginx start cd /Windshaft-cartodb node app.js development & @@ -16,5 +15,4 @@ cd /cartodb source /usr/local/rvm/scripts/rvm bundle exec script/restore_redis bundle exec script/resque > resque.log 2>&1 & -bundle exec rails s -p $PORT - +bundle exec thin start --threaded -p 3000 --threadpool-size 5 diff --git a/template_postgis.sh b/template_postgis.sh index 5122a62..9e4226f 100755 --- a/template_postgis.sh +++ b/template_postgis.sh @@ -3,7 +3,7 @@ # Init script for template postgis # -POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.1.2; +POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.2; createdb -E UTF8 template_postgis; createlang -d template_postgis plpgsql; psql -d postgres -c "UPDATE pg_database SET datistemplate='true' \ @@ -13,4 +13,3 @@ psql -d template_postgis -c "CREATE EXTENSION postgis_topology;" psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;" psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;" psql -c "CREATE EXTENSION plpythonu;" -psql -c "CREATE EXTENSION schema_triggers;"