From d7ab8b6dac54e31384633b69e467d500bcf0f3ba Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Sat, 16 May 2015 13:01:49 +0200 Subject: [PATCH] Put cartodb db services behind nginx. Updated configs to master branch of Cartodb repos. --- README.md | 9 ++- config/CartoDB-dev.js | 15 ++++- config/WS-dev.js | 116 ++++++++++++++++++++++++-------- config/app_config.yml | 40 ++++++----- config/cartodb.nginx.proxy.conf | 30 +++++++++ startup.sh | 2 +- 6 files changed, 166 insertions(+), 46 deletions(-) create mode 100644 config/cartodb.nginx.proxy.conf diff --git a/README.md b/README.md index 81407cd..837feca 100644 --- a/README.md +++ b/README.md @@ -27,5 +27,12 @@ How to run the container: docker run -t -i -p 3000:3000 -p 8080:8080 -p 8181:8181 fleu42/docker-cartodb ``` -You might need to add cartodb.localhost, dev.cartodb.localhost and example.cartodb.localhost to your hosts file. +You need to add `config/cartodb.nginx.proxy.conf` to /etc/nginx/conf.d/. +This will setup a reverse proxy for the CartoDB/imports (3000), SQL Api (8080) and Map api (8181). + +You need to add cartodb.localhost, dev.cartodb.localhost and example.cartodb.localhost to your hosts file. + +Visit http://dev.cartodb.localhost or http://example.cartodb.localhost + Any organization member you create should also be added to your hosts file. + diff --git a/config/CartoDB-dev.js b/config/CartoDB-dev.js index 369d0a9..2a086ae 100644 --- a/config/CartoDB-dev.js +++ b/config/CartoDB-dev.js @@ -1,4 +1,6 @@ -module.exports.base_url = '/api/:version'; +// In case the base_url has a :user param the username will be the one specified in the URL, +// otherwise it will fallback to extract the username from the host header. +module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)'; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various // steps taken for producing the response. @@ -34,6 +36,17 @@ module.exports.db_pool_size = 500; module.exports.db_pool_idleTimeout = 30000; // Milliseconds between idle client checking module.exports.db_pool_reapInterval = 1000; +// max number of bytes for a row, when exceeded the query will throw an error +//module.exports.db_max_row_size = 10 * 1024 * 1024; +// allows to use an object to connect with node-postgres instead of a connection string +//module.exports.db_use_config_object = true; +// requires enabling db_use_config_object=true +// allows to enable/disable keep alive for database connections +// by default is not enabled +//module.exports.db_keep_alive = { +// enabled: true, +// initialDelay: 5000 +//}; module.exports.redis_host = '127.0.0.1'; module.exports.redis_port = 6379; module.exports.redisPool = 50; diff --git a/config/WS-dev.js b/config/WS-dev.js index 050b30f..9c4aca5 100644 --- a/config/WS-dev.js +++ b/config/WS-dev.js @@ -14,13 +14,11 @@ var config = { // Base url for the Templated Maps API // "/api/v1/map/named" is the new API, // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)' + ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' // Base url for the Detached Maps API // "maps" is the the new API, // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)' - // Base url for the Inline Maps and Table Maps API - ,base_url_legacy: '/tiles/:table' + ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' // Maximum number of connections for one process // 128 is a good value with a limit of 1024 open file descriptors @@ -86,9 +84,63 @@ var config = { ,renderer: { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, - metatile: 4, - bufferSize: 64, - statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status + statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mapnik: { + // The size of the pool of internal mapnik renderers + // Check the configuration of uv_threadpool_size to use suitable value + poolSize: 8, + + // Metatile is the number of tiles-per-side that are going + // to be rendered at once. If all of them will be requested + // we'd have saved time. If only one will be used, we'd have + // wasted time. + metatile: 2, + + // Buffer size is the tickness in pixel of a buffer + // around the rendered (meta?)tile. + // + // This is important for labels and other marker that overlap tile boundaries. + // Setting to 128 ensures no render artifacts. + // 64 may have artifacts but is faster. + // Less important if we can turn metatiling on. + bufferSize: 64, + + // SQL queries will be wrapped with ST_SnapToGrid + // Snapping all points of the geometry to a regular grid + snapToGrid: false, + + // SQL queries will be wrapped with ST_ClipByBox2D + // Returning the portion of a geometry falling within a rectangle + // It will only work if snapToGrid is enabled + clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + + limits: { + // Time in milliseconds a render request can take before it fails, some notes: + // - 0 means no render limit + // - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile) + render: 0, + // As the render request will finish even if timed out, whether it should be placed in the internal + // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve + // the same tile will result in an immediate response, however that will use a lot of more application + // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the + // internal cache. + cacheOnTimeout: true + } + }, + http: { + timeout: 2000, // the timeout in ms for a http tile request + proxy: undefined, // the url for a proxy server + whitelist: [ // the whitelist of urlTemplates that can be used + '.*', // will enable any URL + 'http://{s}.example.com/{z}/{x}/{y}.png' + ], + // image to use as placeholder when urlTemplate is not in the whitelist + // if provided the http renderer will use it instead of throw an error + fallbackImage: { + type: 'fs', // 'fs' and 'url' supported + src: __dirname + '/../../assets/default-placeholder.png' + } + } } ,millstone: { // Needs to be writable by server user @@ -120,37 +172,45 @@ var config = { statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd } } - ,sqlapi: { - protocol: 'http', - // If "host" is given, it will be used - // to connect to the SQL-API without a - // DNS lookup - host: '127.0.0.1', - port: 8080, - // The "domain" part will be appended to - // the cartodb username and passed to - // SQL-API requests in the Host HTTP header - domain: 'cartodb.localhost', - version: 'v1', - // Maximum lenght of SQL query for GET - // requests. Longer queries will be sent - // using POST. Defaults to 2048 - max_get_sql_length: 2048, - // Maximum time to wait for a response, - // in milliseconds. Defaults to 100. - timeout: 100 - } ,varnish: { host: 'localhost', - port: 6082, + port: 6082, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses } + // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly + ,fastly: { + // whether the invalidation is enabled or not + enabled: false, + // the fastly api key + apiKey: 'wadus_api_key', + // the service that will get surrogate key invalidation + serviceId: 'wadus_service_id' + } // If useProfiler is true every response will be served with an // X-Tiler-Profile header containing elapsed timing for various // steps taken for producing the response. ,useProfiler:true + // Settings for the health check available at /health + ,health: { + enabled: false, + username: 'localhost', + z: 0, + x: 0, + y: 0 + } + ,disabled_file: 'pids/disabled' + + // Use this as a feature flags enabling/disabling mechanism + ,enabledFeatures: { + // whether it should intercept tile render errors an act based on them, enabled by default. + onTileErrorStrategy: true, + // whether the affected tables for a given SQL must query directly postgresql or use the SQL API + cdbQueryTablesFromPostgres: true + } }; module.exports = config; diff --git a/config/app_config.yml b/config/app_config.yml index 627e060..dcf9b7e 100644 --- a/config/app_config.yml +++ b/config/app_config.yml @@ -5,9 +5,14 @@ defaults: &defaults debug_assets: true mandatory_keys: [layer_opts, sql_api, varnish_management, redis, session_domain] session_domain: '.cartodb.localhost' + # If activated, urls will use usernames in format //SESSION_DOMAIN/user/USERNAME and ignore subdomains if present + subdomainless_urls: false + http_port: # 3000 # nil|integer. HTTP port to use when building urls. Leave empty to use default (80) + https_port: # nil|integer. HTTPS port to use when building urls. Leave empty to use default (443) secret_token: '71c2b25921b84a1cb21c71503ab8fb23' account_host: 'cartodb.localhost:3000' account_path: '/account' + disable_file: '~/disable' watcher: ttl: 60 tiler: @@ -49,18 +54,20 @@ defaults: &defaults tumblr: api_key: '' trackjs: + enabled: false customer: '' + app_keys: + editor: '' + embeds: '' + hubspot: '' common_data: - protocol: 'https' - username: '' - host: '' - api_key: '' - format: 'shp' - generate_every: 86400 - s3_bucket_name: '' - common_data: + protocol: 'https' + username: 'common-data' + host: 'cartodb.com' + format: 'shp' + generate_every: 86400 + explore-api: username: '' - api_key: '' varnish_management: critical: false host: '127.0.0.1' @@ -121,10 +128,10 @@ defaults: &defaults host: "" port: layer_opts: - public_opts: ["type", "active", "query", "opacity", "auto_bound", - "interactivity", "debug", "visible", "tiler_domain", - "tiler_port", "tiler_protocol", "sql_domain", "sql_port", - "sql_protocol", "extra_params", "cdn_url", "table_name", + public_opts: ["type", "active", "query", "opacity", "auto_bound", + "interactivity", "debug", "visible", "tiler_domain", + "tiler_port", "tiler_protocol", "sql_domain", "sql_port", + "sql_protocol", "extra_params", "cdn_url", "table_name", "user_name", "style_version", "tile_style", "query_wrapper"] default_tile_styles: point: "{\n marker-fill: #FF6600;\n marker-opacity: 0.9;\n marker-width: 12;\n marker-line-color: white;\n marker-line-width: 3;\n marker-line-opacity: 0.9;\n marker-placement: point;\n marker-type: ellipse;\n marker-allow-overlap: true;\n}" @@ -178,7 +185,7 @@ defaults: &defaults secret_access_key: "test" assets: s3_bucket_name: "tests" - max_file_size: 5242880 # 5.megabytes + max_file_size: 5242880 # 5.megabytes app_assets: asset_host: "//cartodb-libs.global.ssl.fastly.net/cartodbui" avatars: @@ -186,7 +193,7 @@ defaults: &defaults kinds: ['ghost', 'marker', 'mountain', 'pacman', 'planet', 'stars'] colors: ['green', 'orange', 'red', 'yellow'] dropbox_api_key: "" - gdrive: + gdrive: api_key: "" app_id: "" # This enables a support chat within editor @@ -287,6 +294,7 @@ defaults: &defaults name: 'Nokia hybrid Day' CartoDB: positron_rainbow: + default: true url: 'http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png' subdomains: 'abcd' minZoom: '0' @@ -394,6 +402,7 @@ defaults: &defaults development: <<: *defaults + #http_port: 3000 varnish_management: critical: false host: '127.0.0.1' @@ -406,6 +415,7 @@ development: test: <<: *defaults + http_port: 53716 redis: host: '127.0.0.1' port: 6335 diff --git a/config/cartodb.nginx.proxy.conf b/config/cartodb.nginx.proxy.conf new file mode 100644 index 0000000..63e5865 --- /dev/null +++ b/config/cartodb.nginx.proxy.conf @@ -0,0 +1,30 @@ +server { + server_name cartodb.localhost *.cartodb.localhost; + + location /api/v1/map/named { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://127.0.0.1:8181; + } + + location /api/v1/sql { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://127.0.0.1:8080; + } + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://127.0.0.1:3000; + } + + error_log /var/log/nginx/cartodb_error.log; + access_log /var/log/nginx/cartodb_access.log; +} diff --git a/startup.sh b/startup.sh index 430feb0..1a05ec5 100755 --- a/startup.sh +++ b/startup.sh @@ -15,6 +15,6 @@ node app.js development & cd /cartodb source /usr/local/rvm/scripts/rvm bundle exec script/restore_redis -bundle exec script/resque & +bundle exec script/resque > resque.log 2>&1 & bundle exec rails s -p $PORT