Put cartodb db services behind nginx.

Updated configs to master branch of Cartodb repos.

README.md

@@ -27,5 +27,12 @@ How to run the container:
 docker run -t -i -p 3000:3000 -p 8080:8080 -p 8181:8181 fleu42/docker-cartodb 
 ```
 
-You might need to add cartodb.localhost, dev.cartodb.localhost and example.cartodb.localhost to your hosts file.
+You need to add `config/cartodb.nginx.proxy.conf` to /etc/nginx/conf.d/. 
+This will setup a reverse proxy for the CartoDB/imports (3000), SQL Api (8080) and Map api (8181).
+
+You need to add cartodb.localhost, dev.cartodb.localhost and example.cartodb.localhost to your hosts file.
+
+Visit http://dev.cartodb.localhost or http://example.cartodb.localhost 
+
 Any organization member you create should also be added to your hosts file.
+

config/CartoDB-dev.js

@@ -1,4 +1,6 @@
-module.exports.base_url     = '/api/:version';
+// In case the base_url has a :user param the username will be the one specified in the URL,
+// otherwise it will fallback to extract the username from the host header.
+module.exports.base_url     = '(?:/api/:version|/user/:user/api/:version)';
 // If useProfiler is true every response will be served with an
 // X-SQLAPI-Profile header containing elapsed timing for various
 // steps taken for producing the response.
@@ -34,6 +36,17 @@ module.exports.db_pool_size = 500;
 module.exports.db_pool_idleTimeout = 30000;
 // Milliseconds between idle client checking
 module.exports.db_pool_reapInterval = 1000;
+// max number of bytes for a row, when exceeded the query will throw an error
+//module.exports.db_max_row_size = 10 * 1024 * 1024;
+// allows to use an object to connect with node-postgres instead of a connection string
+//module.exports.db_use_config_object = true;
+// requires enabling db_use_config_object=true
+// allows to enable/disable keep alive for database connections
+// by default is not enabled
+//module.exports.db_keep_alive = {
+//    enabled: true,
+//    initialDelay: 5000
+//};
 module.exports.redis_host   = '127.0.0.1';
 module.exports.redis_port   = 6379;
 module.exports.redisPool    = 50;

config/WS-dev.js

@@ -14,13 +14,11 @@ var config = {
     // Base url for the Templated Maps API
     // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
+    ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
-    // Base url for the Inline Maps and Table Maps API
-    ,base_url_legacy: '/tiles/:table'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
@@ -86,9 +84,63 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
+      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
-      bufferSize: 64,
+      mapnik: {
-      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
+          // The size of the pool of internal mapnik renderers
+          // Check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          }
+      },
+      http: {
+          timeout: 2000, // the timeout in ms for a http tile request
+          proxy: undefined, // the url for a proxy server
+          whitelist: [ // the whitelist of urlTemplates that can be used
+              '.*', // will enable any URL
+              'http://{s}.example.com/{z}/{x}/{y}.png'
+          ],
+          // image to use as placeholder when urlTemplate is not in the whitelist
+          // if provided the http renderer will use it instead of throw an error
+          fallbackImage: {
+              type: 'fs', // 'fs' and 'url' supported
+              src: __dirname + '/../../assets/default-placeholder.png'
+          }
+      }
     }
     ,millstone: {
         // Needs to be writable by server user
@@ -120,37 +172,45 @@ var config = {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
         }
     }
-    ,sqlapi: {
-        protocol: 'http',
-        // If "host" is given, it will be used
-        // to connect to the SQL-API without a
-        // DNS lookup
-        host: '127.0.0.1',
-        port: 8080,
-        // The "domain" part will be appended to 
-        // the cartodb username and passed to
-        // SQL-API requests in the Host HTTP header
-        domain: 'cartodb.localhost',
-        version: 'v1',
-        // Maximum lenght of SQL query for GET
-        // requests. Longer queries will be sent
-        // using POST. Defaults to 2048
-        max_get_sql_length: 2048,
-        // Maximum time to wait for a response,
-        // in milliseconds. Defaults to 100.
-        timeout: 100
-    }
     ,varnish: {
         host: 'localhost',
-        port: 6082,
+        port: 6082, // the por for the telnet interface where varnish is listening to
+        http_port: 6081, // the port for the HTTP interface where varnish is listening to
+        purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
+    // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly
+    ,fastly: {
+        // whether the invalidation is enabled or not
+        enabled: false,
+        // the fastly api key
+        apiKey: 'wadus_api_key',
+        // the service that will get surrogate key invalidation
+        serviceId: 'wadus_service_id'
+    }
     // If useProfiler is true every response will be served with an
     // X-Tiler-Profile header containing elapsed timing for various 
     // steps taken for producing the response.
     ,useProfiler:true
+    // Settings for the health check available at /health
+    ,health: {
+      enabled: false,
+      username: 'localhost',
+      z: 0,
+      x: 0,
+      y: 0
+    }
+    ,disabled_file: 'pids/disabled'
+
+    // Use this as a feature flags enabling/disabling mechanism
+    ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
+        // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
+        cdbQueryTablesFromPostgres: true
+    }
 };
 
 module.exports = config;

config/app_config.yml

@@ -5,9 +5,14 @@ defaults: &defaults
   debug_assets: true
   mandatory_keys:     [layer_opts, sql_api, varnish_management, redis, session_domain]
   session_domain:     '.cartodb.localhost'
+  # If activated, urls will use usernames in format //SESSION_DOMAIN/user/USERNAME and ignore subdomains if present
+  subdomainless_urls: false
+  http_port:           # 3000 # nil|integer. HTTP port to use when building urls. Leave empty to use default (80)
+  https_port:          # nil|integer. HTTPS port to use when building urls. Leave empty to use default (443)
   secret_token:       '71c2b25921b84a1cb21c71503ab8fb23'
   account_host:       'cartodb.localhost:3000'
   account_path:       '/account'
+  disable_file:       '~/disable'
   watcher:
     ttl:              60
   tiler:
@@ -49,18 +54,20 @@ defaults: &defaults
   tumblr:
     api_key: ''
   trackjs:
+    enabled: false
     customer: ''
+    app_keys:
+      editor: ''
+      embeds: ''
+  hubspot: ''
   common_data:
-       protocol: 'https'
+    protocol: 'https'
-       username: ''
+    username: 'common-data'
-       host: ''
+    host: 'cartodb.com'
-       api_key: ''
+    format: 'shp'
-       format: 'shp'
+    generate_every: 86400
-       generate_every: 86400
+  explore-api:
-       s3_bucket_name: ''
-  common_data:
     username: ''
-    api_key: ''
   varnish_management:
     critical: false
     host: '127.0.0.1'
@@ -287,6 +294,7 @@ defaults: &defaults
         name: 'Nokia hybrid Day'
     CartoDB:
       positron_rainbow:
+        default: true
         url: 'http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png'
         subdomains: 'abcd'
         minZoom: '0'
@@ -394,6 +402,7 @@ defaults: &defaults
 
 development:
   <<: *defaults
+  #http_port: 3000
   varnish_management:
     critical: false
     host: '127.0.0.1'
@@ -406,6 +415,7 @@ development:
 
 test:
   <<: *defaults
+  http_port: 53716
   redis:
     host: '127.0.0.1'
     port: 6335

config/cartodb.nginx.proxy.conf

@@ -0,0 +1,30 @@
+server {
+  server_name cartodb.localhost *.cartodb.localhost;
+
+  location /api/v1/map/named {
+    proxy_set_header        Host $host;
+    proxy_set_header        X-Real-IP $remote_addr;
+    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header        X-Forwarded-Proto $scheme;
+    proxy_pass http://127.0.0.1:8181;
+  }
+
+  location /api/v1/sql {
+    proxy_set_header        Host $host;
+    proxy_set_header        X-Real-IP $remote_addr;
+    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header        X-Forwarded-Proto $scheme;
+    proxy_pass http://127.0.0.1:8080;
+  }
+
+  location / {
+    proxy_set_header        Host $host;
+    proxy_set_header        X-Real-IP $remote_addr;
+    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header        X-Forwarded-Proto $scheme;
+    proxy_pass http://127.0.0.1:3000;
+  }
+
+  error_log /var/log/nginx/cartodb_error.log;
+  access_log /var/log/nginx/cartodb_access.log;
+}

startup.sh

@@ -15,6 +15,6 @@ node app.js development &
 cd /cartodb
 source /usr/local/rvm/scripts/rvm
 bundle exec script/restore_redis
-bundle exec script/resque &
+bundle exec script/resque > resque.log 2>&1 &
 bundle exec rails s -p $PORT