diff --git a/config/CartoDB-dev.js b/config/CartoDB-dev.js index 2a086ae..1f020b4 100644 --- a/config/CartoDB-dev.js +++ b/config/CartoDB-dev.js @@ -5,7 +5,7 @@ module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)'; // X-SQLAPI-Profile header containing elapsed timing for various // steps taken for producing the response. module.exports.useProfiler = true; -module.exports.log_format = '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])'; +module.exports.log_format = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])'; // If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal module.exports.log_filename = 'logs/cartodb-sql-api.log'; @@ -28,9 +28,11 @@ module.exports.db_pubuser = 'publicuser'; module.exports.db_pubuser_pass = 'public'; module.exports.db_host = 'localhost'; module.exports.db_port = '5432'; +module.exports.db_batch_port = '5432'; +module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours // Max database connections in the pool // Subsequent connections will wait for a free slot. -// NOTE: not used by OGR-mediated accesses +// NOTE: not used by OGR-mediated accesses module.exports.db_pool_size = 500; // Milliseconds before a connection is removed from pool module.exports.db_pool_idleTimeout = 30000; @@ -59,6 +61,8 @@ module.exports.tableCacheMax = 8192; module.exports.tableCacheMaxAge = 1000*60*10; // Temporary directory, make sure it is writable by server user module.exports.tmpDir = '/tmp'; +// change ogr2ogr command or path +module.exports.ogr2ogrCommand = 'ogr2ogr'; // Optional statsd support module.exports.statsd = { host: 'localhost', @@ -72,3 +76,4 @@ module.exports.health = { username: 'development', query: 'select 1' }; +module.exports.disabled_file = 'pids/disabled'; diff --git a/config/WS-dev.js b/config/WS-dev.js index 9c4aca5..3427208 100644 --- a/config/WS-dev.js +++ b/config/WS-dev.js @@ -2,6 +2,9 @@ var config = { environment: 'development' ,port: 8181 ,host: '0.0.0.0' + // Size of the threadpool which can be used to run user code and get notified in the loop thread + // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). + // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. @@ -10,7 +13,7 @@ var config = { // Base URLs for the APIs // // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // + // // Base url for the Templated Maps API // "/api/v1/map/named" is the new API, // "/tiles/template" is for compatibility with versions up to 1.6.x @@ -30,7 +33,7 @@ var config = { // to be able to navigate the map without a reload ? // Defaults to 7200 (2 hours) ,mapConfigTTL: 7200 - // idle socket timeout, in milliseconds + // idle socket timeout, in milliseconds ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: false @@ -38,7 +41,7 @@ var config = { // If log_filename is given logs will be written // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - ,log_filename: 'logs/node-windshaft.log' + ,log_filename: undefined // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>' @@ -59,6 +62,7 @@ var config = { extent: "-180,-90,180,90", srid: 4326, */ + // max number of rows to return when querying data, 0 means no limit row_limit: 65535, simplify_geometries: true, use_overviews: true, // use overviews to retrieve raster @@ -86,8 +90,10 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mapnik: { - // The size of the pool of internal mapnik renderers - // Check the configuration of uv_threadpool_size to use suitable value + // The size of the pool of internal mapnik backend + // This pool size is per mapnik renderer created in Windshaft's RendererFactory + // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js + // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, // Metatile is the number of tiles-per-side that are going @@ -96,6 +102,23 @@ var config = { // wasted time. metatile: 2, + // tilelive-mapnik uses an internal cache to store tiles/grids + // generated when using metatile. This options allow to tune + // the behaviour for that internal cache. + metatileCache: { + // Time an object must stay in the cache until is removed + ttl: 0, + // Whether an object must be removed after the first hit + // Usually you want to use `true` here when ttl>0. + deleteOnHit: false + }, + + // Override metatile behaviour depending on the format + formatMetatile: { + png: 2, + 'grid.json': 1 + }, + // Buffer size is the tickness in pixel of a buffer // around the rendered (meta?)tile. // @@ -125,7 +148,28 @@ var config = { // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the // internal cache. cacheOnTimeout: true + }, + + geojson: { + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + }, + + // SQL queries will be wrapped with ST_ClipByBox2D + // Returning the portion of a geometry falling within a rectangle + // It will only work if snapToGrid is enabled + clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + // geometries will be simplified using ST_RemoveRepeatedPoints + // which cost is no more expensive than snapping and results are + // much closer to the original geometry + removeRepeatedPoints: false // this requires postgis >=2.2 } + }, http: { timeout: 2000, // the timeout in ms for a http tile request @@ -140,8 +184,34 @@ var config = { type: 'fs', // 'fs' and 'url' supported src: __dirname + '/../../assets/default-placeholder.png' } + }, + torque: { + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: false, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' @@ -170,7 +240,16 @@ var config = { }, emitter: { statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd - } + }, + unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 + noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading + } + // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options + ,httpAgent: { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 25, + maxFreeSockets: 256 } ,varnish: { host: 'localhost', @@ -191,7 +270,7 @@ var config = { serviceId: 'wadus_service_id' } // If useProfiler is true every response will be served with an - // X-Tiler-Profile header containing elapsed timing for various + // X-Tiler-Profile header containing elapsed timing for various // steps taken for producing the response. ,useProfiler:true // Settings for the health check available at /health @@ -209,7 +288,10 @@ var config = { // whether it should intercept tile render errors an act based on them, enabled by default. onTileErrorStrategy: true, // whether the affected tables for a given SQL must query directly postgresql or use the SQL API - cdbQueryTablesFromPostgres: true + cdbQueryTablesFromPostgres: true, + // whether in mapconfig is available stats & metadata for each layer + layerMetadata: true + } }; diff --git a/config/app_config.yml b/config/app_config.yml index d382fe9..9b1965e 100644 --- a/config/app_config.yml +++ b/config/app_config.yml @@ -1,7 +1,12 @@ defaults: &defaults + # If uncommented, most images like static map previews and twitter card image urls will use this CDN urls + #cdn_url: + # http: "http.cdn.host" + # https: "https.cdn.host" + http_client_logs: true ogr2ogr: binary: 'which ogr2ogr2' - csv_guessing: false + csv_guessing: true debug_assets: true mandatory_keys: [layer_opts, sql_api, varnish_management, redis, session_domain] session_domain: 'cartodb.localhost' @@ -12,6 +17,7 @@ defaults: &defaults secret_token: '71c2b25921b84a1cb21c71503ab8fb23' account_host: 'cartodb.localhost:3000' account_path: '/account' + vizjson_cache_domains: ['.localhost'] data_library: username: 'common-data' path: '/data-library' @@ -53,8 +59,7 @@ defaults: &defaults primary: '' embeds: '' domain: '' - rollbar: - token: '' + rollbar_api_key: '' tumblr: api_key: '' trackjs: @@ -85,6 +90,8 @@ defaults: &defaults connect_dataset: '' create_map: '' export_table: '' + export_map: '' + export_public_map: '' select_wms: '' color_basemap: '' pattern_basemap: '' @@ -101,14 +108,15 @@ defaults: &defaults common_data: protocol: 'https' username: 'common-data' - base_url: 'https://common-data.cartodb.com' + base_url: 'https://common-data.carto.com' format: 'shp' + generate_every: 86400 explore_api: username: '' reports: mail_to: '' mailer: - from: 'cartodb.com ' + from: 'cartodb.localhost ' address: '' port: '' user_name: '' @@ -135,7 +143,9 @@ defaults: &defaults redis: host: '127.0.0.1' port: 6379 - timeout: 20 + connect_timeout: 2 + read_timeout: 3 + write_timeout: 5 databases: tables_metadata: 0 api_credentials: 3 @@ -165,12 +175,27 @@ defaults: &defaults base_url: '' api_key: '' table_name: '' - api: - host: 'cartodb.localhost' - port: '8787' - dbname: 'geocoder' - user: 'geocoder' + user_migrator: + user_exports_folder: '/tmp/user_exports' + user_imports_folder: '/tmp/user_imports' + s3: + access_key_id: '' + secret_access_key: '' + bucket_name: '' + url_ttl: 7200 + async_long_uploads: false + uploads_path: 'public/uploads' + exporter: + exporter_temporal_folder: '/tmp/exporter' + s3: + access_key_id: '' + secret_access_key: '' + bucket_name: '' + url_ttl: 7200 + async_long_uploads: false + uploads_path: 'public/uploads' # including 'uploads' assumes public path. Absolute path example: /tmp/exports/downloads importer: + blacklisted_ip_addr: [] content_guessing: # Depends on geocoding enabled: false # Disabled if false or not present threshold: 0.9 # 90% or more matches @@ -182,11 +207,14 @@ defaults: &defaults bucket_name: url_ttl: async_long_uploads: false + proxy_uri: + use_ssl: True + unp_temporal_folder: '/tmp/imports/' # It must end in `/uploads` and be accessible via HTTP, if relative will default to Rails.Root/#{uploads_path} uploads_path: 'public/uploads' error_track: - url: 'https://viz2.cartodb.com/api/v1/sql' + url: 'https://viz2.carto.com/api/v1/sql' percent_users: 10 # graphite endpoint used to post frontend stats graphite_public: @@ -207,7 +235,7 @@ defaults: &defaults data: kind: "carto" options: - # attribution: 'CartoDB attribution' + # attribution: 'CARTO attribution' query: "" opacity: 0.99 auto_bound: false @@ -235,12 +263,12 @@ defaults: &defaults access_key_id: "test" secret_access_key: "test" assets: - # store assets locally instead of s3 - # s3_bucket_name: "tests" + s3_bucket_name: "tests" max_file_size: 5242880 # 5.megabytes app_assets: asset_host: "//cartodb-libs.global.ssl.fastly.net/cartodbui" avatars: + gravatar_enabled: true base_url: '//cartodb-libs.global.ssl.fastly.net/cartodbui/assets/unversioned/images/avatars' kinds: ['ghost', 'marker', 'mountain', 'pacman', 'planet', 'star'] colors: ['green', 'orange', 'red', 'yellow'] @@ -267,11 +295,14 @@ defaults: &defaults application_name: '' client_id: '' client_secret: '' - callback_url: 'http://cartodb.com' - google_plus: - client_id: '' - cookie_policy: 'single_host_origin' - #cookie_policy: 'https://cartodb-staging.com' + callback_url: 'https://carto.com' + # google_plus: + # client_id: '' + # cookie_policy: 'single_host_origin' + # cookie_policy: 'https://cartodb-staging.com' + github: + client_id: '' + client_secret: '' dropbox: app_key: '' app_secret: '' @@ -316,13 +347,6 @@ defaults: &defaults ratelimit_concurrency: 3 ratelimit_ttl: 4 ratelimit_wait_secs: 0.1 - feature_flags: - ghost_tables: - restricted: true - rainbow_maps: - restricted: false - user_feature_flags: - rambo: ['ghost_tables'] basemaps: Nokia: normal_day: @@ -353,24 +377,44 @@ defaults: &defaults className: "nokia_hybrid_day" attribution: "©2012 Nokia Terms of use" name: 'Nokia hybrid Day' - CartoDB: - positron_rainbow: + CARTO: + positron_rainbow_labels: default: true - url: 'http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png' + url: 'http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png' subdomains: 'abcd' minZoom: '0' maxZoom: '18' name: 'Positron' + className: 'positron_rainbow_labels' + attribution: '© OpenStreetMap contributors © CARTO' + labels: + url: 'http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png' + dark_matter_rainbow_labels: + url: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png' + subdomains: 'abcd' + minZoom: '0' + maxZoom: '18' + name: 'Dark matter' + className: 'dark_matter_rainbow_labels' + attribution: '© OpenStreetMap contributors © CARTO' + labels: + url: 'http://{s}.basemaps.cartocdn.com/dark_only_labels/{z}/{x}/{y}.png' + positron_rainbow: + url: 'http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png' + subdomains: 'abcd' + minZoom: '0' + maxZoom: '18' + name: 'Positron (labels below)' className: 'positron_rainbow' - attribution: '© OpenStreetMap contributors © CartoDB' + attribution: '© OpenStreetMap contributors © CARTO' dark_matter_rainbow: url: 'http://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png' subdomains: 'abcd' minZoom: '0' maxZoom: '18' - name: 'Dark matter' + name: 'Dark matter (labels below)' className: 'dark_matter_rainbow' - attribution: '© OpenStreetMap contributors © CartoDB' + attribution: '© OpenStreetMap contributors © CARTO' positron_lite_rainbow: url: 'http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png' subdomains: 'abcd' @@ -378,7 +422,7 @@ defaults: &defaults maxZoom: '18' name: 'Positron (lite)' className: 'positron_lite_rainbow' - attribution: '© OpenStreetMap contributors © CartoDB' + attribution: '© OpenStreetMap contributors © CARTO' dark_matter_lite_rainbow: url: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png' subdomains: 'abcd' @@ -386,13 +430,13 @@ defaults: &defaults maxZoom: '18' name: 'Dark matter (lite)' className: 'dark_matter_lite_rainbow' - attribution: '© OpenStreetMap contributors © CartoDB' + attribution: '© OpenStreetMap contributors © CARTO' eco_cartodb: url: 'https://cartocdn_{s}.global.ssl.fastly.net/base-antique/{z}/{x}/{y}.png' subdomains: 'abcd' minZoom: '0' maxZoom: '10' - name: 'CartoDB World Eco' + name: 'CARTO World Eco' className: 'eco_cartodb' attribution: '' flat_blue: @@ -400,7 +444,7 @@ defaults: &defaults subdomains: 'abcd' minZoom: '0' maxZoom: '10' - name: 'CartoDB World Flat Blue' + name: 'CARTO World Flat Blue' className: 'flat_blue' attribution: '' midnight_cartodb: @@ -408,9 +452,17 @@ defaults: &defaults subdomains: 'abcd' minZoom: '0' maxZoom: '10' - name: 'CartoDB World Midnight Commander' + name: 'CARTO World Midnight Commander' className: 'midnight_cartodb' attribution: '' + antique_cartodb: + url: 'https://cartocdn_{s}.global.ssl.fastly.net/base-antique/{z}/{x}/{y}.png' + subdomains: 'abcd' + minZoom: 0 + maxZoom: 10 + name: 'CARTO World Antique' + className: 'antique_cartodb' + attribution: '' Stamen: toner_stamen_labels: url: 'https://stamen-tiles-{s}.a.ssl.fastly.net/toner-background/{z}/{x}/{y}.png' @@ -470,6 +522,10 @@ defaults: &defaults name: 'Watercolor' className: 'watercolor_stamen' attribution: 'Map tiles by Stamen Design, under CC BY 3.0. Data by OpenStreetMap, under ODbL.' + overviews: + min_rows: 2000000 + statement_timeout: 1800000 + tolerance_px: 1.0 development: <<: *defaults