From 90d0b23441219130d72c1d2a9237a45a87f470a3 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 24 Jun 2015 15:43:02 +0200 Subject: [PATCH 01/57] Use CDB_QueryTablesText instead of CDB_QueryTables This avoids trouble with len(schema.table_name) > 63 See https://github.com/CartoDB/cartodb-postgresql/issues/86 --- lib/cartodb/api/query_tables_api.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/api/query_tables_api.js b/lib/cartodb/api/query_tables_api.js index fc0e2932..829d82cd 100644 --- a/lib/cartodb/api/query_tables_api.js +++ b/lib/cartodb/api/query_tables_api.js @@ -14,7 +14,7 @@ module.exports = QueryTablesApi; QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) { - var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)'; + var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)'; this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback); }; @@ -25,9 +25,9 @@ function handleAffectedTablesInQueryRows(err, rows, callback) { callback(new Error('could not fetch source tables: ' + msg)); return; } - var qtables = rows[0].cdb_querytables; - var tableNames = qtables.split(/^\{(.*)\}$/)[1]; - tableNames = tableNames ? tableNames.split(',') : []; + + // This is an Array, so no need to split into parts + var tableNames = rows[0].cdb_querytablestext; callback(null, tableNames); } @@ -35,7 +35,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam var query = [ 'WITH querytables AS (', - 'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames', + 'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames', ')', 'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max', 'FROM CDB_TableMetadata m', @@ -54,8 +54,8 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { var result = rows[0]; - var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1]; - tableNames = tableNames ? tableNames.split(',') : []; + // This is an Array, so no need to split into parts + var tableNames = result.tablenames; var lastUpdatedTime = result.max || 0; From 2715f47a224cf21b94867f5fba1abd9e2f40c63b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 24 Jun 2015 19:07:41 +0200 Subject: [PATCH 02/57] Points CDB_QueryTables script to the branch with CDB_QueryTablesText --- test/support/prepare_db.sh | 2 +- test/support/sql/CDB_QueryTables.sql | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 8185da4d..3b827f40 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -80,7 +80,7 @@ if test x"$PREPARE_PGSQL" = xyes; then psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB} curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql + curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/86-CDB_QueryTables-fix-long-names/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 diff --git a/test/support/sql/CDB_QueryTables.sql b/test/support/sql/CDB_QueryTables.sql index cd8b51b4..ac61281d 100644 --- a/test/support/sql/CDB_QueryTables.sql +++ b/test/support/sql/CDB_QueryTables.sql @@ -2,12 +2,12 @@ -- -- Requires PostgreSQL 9.x+ -- -CREATE OR REPLACE FUNCTION CDB_QueryTables(query text) -RETURNS name[] +CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text) +RETURNS text[] AS $$ DECLARE exp XML; - tables NAME[]; + tables text[]; rec RECORD; rec2 RECORD; BEGIN @@ -41,11 +41,11 @@ BEGIN xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x, xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s ) - SELECT unnest(x)::name as p, unnest(s)::name as sc from inp + SELECT unnest(x) as p, unnest(s) as sc from inp LOOP -- RAISE DEBUG 'tab: %', rec2.p; -- RAISE DEBUG 'sc: %', rec2.sc; - tables := array_append(tables, (rec2.sc || '.' || rec2.p)::name); + tables := array_append(tables, (rec2.sc || '.' || rec2.p)); END LOOP; -- RAISE DEBUG 'Tables: %', tables; @@ -65,3 +65,14 @@ BEGIN return tables; END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; + + +-- Keep CDB_QueryTables with same signature for backwards compatibility. +-- It should probably be removed in the future. +CREATE OR REPLACE FUNCTION CDB_QueryTables(query text) +RETURNS name[] +AS $$ +BEGIN + RETURN CDB_QueryTablesText(query)::name[]; +END +$$ LANGUAGE 'plpgsql' VOLATILE STRICT; From 9e30f05e7d0133de3fa7aa349915d8a8f1587c2d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 29 Jun 2015 16:46:07 +0200 Subject: [PATCH 03/57] Reverts to use cdb branch as is already published --- test/support/prepare_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 3b827f40..8185da4d 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -80,7 +80,7 @@ if test x"$PREPARE_PGSQL" = xyes; then psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB} curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/86-CDB_QueryTables-fix-long-names/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql + curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 From 9a393fa7931c141623c876b11d70bfc8a97c04dc Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 7 Jul 2015 12:27:09 +0200 Subject: [PATCH 04/57] Adds some notes about uv_threadpool_size and mapnik renderer pool size --- config/environments/development.js.example | 9 +++++++-- config/environments/production.js.example | 9 +++++++-- config/environments/staging.js.example | 9 +++++++-- config/environments/test.js.example | 9 +++++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 3fcf78f9..37c09037 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -2,6 +2,9 @@ var config = { environment: 'development' ,port: 8181 ,host: '127.0.0.1' + // Size of the threadpool which can be used to run user code and get notified in the loop thread + // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). + // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. @@ -86,8 +89,10 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mapnik: { - // The size of the pool of internal mapnik renderers - // Check the configuration of uv_threadpool_size to use suitable value + // The size of the pool of internal mapnik backend + // This pool size is per mapnik renderer created in Windshaft's RendererFactory + // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js + // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, // Metatile is the number of tiles-per-side that are going diff --git a/config/environments/production.js.example b/config/environments/production.js.example index c0a63be9..64e0e26e 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -2,6 +2,9 @@ var config = { environment: 'production' ,port: 8181 ,host: '127.0.0.1' + // Size of the threadpool which can be used to run user code and get notified in the loop thread + // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). + // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. @@ -80,8 +83,10 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mapnik: { - // The size of the pool of internal mapnik renderers - // Check the configuration of uv_threadpool_size to use suitable value + // The size of the pool of internal mapnik backend + // This pool size is per mapnik renderer created in Windshaft's RendererFactory + // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js + // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, // Metatile is the number of tiles-per-side that are going diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 69fa635b..05589ce7 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -2,6 +2,9 @@ var config = { environment: 'production' ,port: 8181 ,host: '127.0.0.1' + // Size of the threadpool which can be used to run user code and get notified in the loop thread + // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). + // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. @@ -80,8 +83,10 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mapnik: { - // The size of the pool of internal mapnik renderers - // Check the configuration of uv_threadpool_size to use suitable value + // The size of the pool of internal mapnik backend + // This pool size is per mapnik renderer created in Windshaft's RendererFactory + // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js + // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, // Metatile is the number of tiles-per-side that are going diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 5597509a..1d8ea9da 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -2,6 +2,9 @@ var config = { environment: 'test' ,port: 8888 ,host: '127.0.0.1' + // Size of the threadpool which can be used to run user code and get notified in the loop thread + // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). + // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. @@ -80,8 +83,10 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mapnik: { - // The size of the pool of internal mapnik renderers - // Check the configuration of uv_threadpool_size to use suitable value + // The size of the pool of internal mapnik backend + // This pool size is per mapnik renderer created in Windshaft's RendererFactory + // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js + // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, // Metatile is the number of tiles-per-side that are going From 962ac97433d11fd71ec4f45877f41162456ed073 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 12:52:12 +0200 Subject: [PATCH 05/57] regenerate npm-shrinkwrap.json --- npm-shrinkwrap.json | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 0e1a88c9..5d41b9e8 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -310,9 +310,9 @@ "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz", "dependencies": { "bluebird": { - "version": "2.9.32", + "version": "2.9.33", "from": "bluebird@^2.9.30", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.32.tgz" + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.33.tgz" }, "chalk": { "version": "1.1.0", @@ -2705,39 +2705,6 @@ } } }, - "redis-mpool": { - "version": "0.4.0", - "from": "redis-mpool@~0.4.0", - "dependencies": { - "generic-pool": { - "version": "2.1.1", - "from": "generic-pool@~2.1.1", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz" - }, - "redis": { - "version": "0.12.1", - "from": "redis@~0.12.1", - "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz" - }, - "hiredis": { - "version": "0.1.17", - "from": "hiredis@~0.1.17", - "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.1.17.tgz", - "dependencies": { - "bindings": { - "version": "1.2.1", - "from": "bindings@*", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz" - }, - "nan": { - "version": "1.1.2", - "from": "nan@~1.1.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-1.1.2.tgz" - } - } - } - } - }, "carto": { "version": "0.15.1-cdb1", "from": "https://github.com/CartoDB/carto/tarball/0.15.1-cdb1", From b05701be615d14add3e03f58f3353d4ba23bfdc8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 15:05:03 +0200 Subject: [PATCH 06/57] Authentication/Authorization moves to its own entity --- lib/cartodb/api/auth_api.js | 141 ++++++++++++++++++++ lib/cartodb/controllers/named_maps_admin.js | 19 ++- lib/cartodb/server.js | 123 +---------------- 3 files changed, 159 insertions(+), 124 deletions(-) create mode 100644 lib/cartodb/api/auth_api.js diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js new file mode 100644 index 00000000..5b20cbf4 --- /dev/null +++ b/lib/cartodb/api/auth_api.js @@ -0,0 +1,141 @@ +var assert = require('assert'); +var step = require('step'); + +/** + * + * @param {PgConnection} pgConnection + * @param metadataBackend + * @param {MapStore} mapStore + * @param {TemplateMaps} templateMaps + * @constructor + * @type {AuthApi} + */ +function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) { + this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; + this.mapStore = mapStore; + this.templateMaps = templateMaps; +} + +module.exports = AuthApi; + +// Check if a request is authorized by a signer +// +// @param req express request object +// @param callback function(err, signed_by) signed_by will be +// null if the request is not signed by anyone +// or will be a string cartodb username otherwise. +// +AuthApi.prototype.authorizedBySigner = function(req, callback) { + if ( ! req.params.token || ! req.params.signer ) { + return callback(null, false); // no signer requested + } + + var self = this; + + var layergroup_id = req.params.token; + var auth_token = req.params.auth_token; + + this.mapStore.load(layergroup_id, function(err, mapConfig) { + if (err) { + return callback(err); + } + + var authorized = self.templateMaps.isAuthorized(mapConfig.obj().template, auth_token); + + return callback(null, authorized); + }); +}; + +// Check if a request is authorized by api_key +// +// @param user +// @param req express request object +// @param callback function(err, authorized) +// NOTE: authorized is expected to be 0 or 1 (integer) +// +AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) { + var givenKey = req.query.api_key || req.query.map_key; + if ( ! givenKey && req.body ) { + // check also in request body + givenKey = req.body.api_key || req.body.map_key; + } + if ( ! givenKey ) { + return callback(null, 0); // no api key, no authorization... + } + + var self = this; + + step( + function () { + self.metadataBackend.getUserMapKey(user, this); + }, + function checkApiKey(err, val){ + assert.ifError(err); + return val && givenKey == val; + }, + function finish(err, authorized) { + callback(err, authorized); + } + ); +}; + +/** + * Check access authorization + * + * @param req - standard req object. Importantly contains table and host information + * @param callback function(err, allowed) is access allowed not? + */ +AuthApi.prototype.authorize = function(req, callback) { + var self = this; + var user = req.context.user; + + step( + function () { + self.authorizedByAPIKey(user, req, this); + }, + function checkApiKey(err, authorized){ + if (req.profiler) { + req.profiler.done('authorizedByAPIKey'); + } + assert.ifError(err); + + // if not authorized by api_key, continue + if (!authorized) { + // not authorized by api_key, check if authorized by signer + return self.authorizedBySigner(req, this); + } + + // authorized by api key, login as the given username and stop + self.pgConnection.setDBAuth(user, req.params, function(err) { + callback(err, true); // authorized (or error) + }); + }, + function checkSignAuthorized(err, authorized) { + if (err) { + return callback(err); + } + + if ( ! authorized ) { + // request not authorized by signer. + + // if no signer name was given, let dbparams and + // PostgreSQL do the rest. + // + if ( ! req.params.signer ) { + return callback(null, true); // authorized so far + } + + // if signer name was given, return no authorization + return callback(null, false); + } + + self.pgConnection.setDBAuth(user, req.params, function(err) { + if (req.profiler) { + req.profiler.done('setDBAuth'); + } + callback(err, true); // authorized (or error) + }); + } + ); +}; diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index db92f3ae..e0b9c035 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -5,9 +5,16 @@ var templateName = require('../backends/template_maps').templateName; var cors = require('../middleware/cors'); -function NamedMapsAdminController(app, templateMaps) { +/** + * @param app + * @param {TemplateMaps} templateMaps + * @param {AuthApi} authApi + * @constructor + */ +function NamedMapsAdminController(app, templateMaps, authApi) { this.app = app; this.templateMaps = templateMaps; + this.authApi = authApi; } module.exports = NamedMapsAdminController; @@ -28,7 +35,7 @@ NamedMapsAdminController.prototype.create = function(req, res) { step( function checkPerms(){ - self.app.authorizedByAPIKey(cdbuser, req, this); + self.authApi.authorizedByAPIKey(cdbuser, req, this); }, function addTemplate(err, authenticated) { assert.ifError(err); @@ -53,7 +60,7 @@ NamedMapsAdminController.prototype.update = function(req, res) { var tpl_id; step( function checkPerms(){ - self.app.authorizedByAPIKey(cdbuser, req, this); + self.authApi.authorizedByAPIKey(cdbuser, req, this); }, function updateTemplate(err, authenticated) { assert.ifError(err); @@ -84,7 +91,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) { var tpl_id; step( function checkPerms(){ - self.app.authorizedByAPIKey(cdbuser, req, this); + self.authApi.authorizedByAPIKey(cdbuser, req, this); }, function getTemplate(err, authenticated) { assert.ifError(err); @@ -120,7 +127,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) { var tpl_id; step( function checkPerms(){ - self.app.authorizedByAPIKey(cdbuser, req, this); + self.authApi.authorizedByAPIKey(cdbuser, req, this); }, function deleteTemplate(err, authenticated) { assert.ifError(err); @@ -147,7 +154,7 @@ NamedMapsAdminController.prototype.list = function(req, res) { step( function checkPerms(){ - self.app.authorizedByAPIKey(cdbuser, req, this); + self.authApi.authorizedByAPIKey(cdbuser, req, this); }, function listTemplates(err, authenticated) { assert.ifError(err); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index ffd71219..fdc028ce 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -19,6 +19,7 @@ var mapnik = windshaft.mapnik; var TemplateMaps = require('./backends/template_maps.js'); var QueryTablesApi = require('./api/query_tables_api'); var UserLimitsApi = require('./api/user_limits_api'); +var AuthApi = require('./api/auth_api'); var PgQueryRunner = require('./backends/pg_query_runner'); var PgConnection = require('./backends/pg_connection'); @@ -167,6 +168,8 @@ module.exports = function(serverOptions) { var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); + app.findStatusCode = function(err) { var statusCode; if ( err.http_status ) { @@ -220,7 +223,7 @@ module.exports = function(serverOptions) { userLimitsApi ).register(app); - new controller.NamedMapsAdmin(app, templateMaps).register(app); + new controller.NamedMapsAdmin(app, templateMaps, authApi).register(app); new controller.ServerInfo().register(app); @@ -453,7 +456,7 @@ module.exports = function(serverOptions) { step( function getPrivacy(){ - app.authorize(req, this); + authApi.authorize(req, this); }, function gatekeep(err, authorized){ if (req.profiler) { @@ -626,122 +629,6 @@ module.exports = function(serverOptions) { }); }; - // Check if a request is authorized by a signer - // - // @param req express request object - // @param callback function(err, signed_by) signed_by will be - // null if the request is not signed by anyone - // or will be a string cartodb username otherwise. - // - app.authorizedBySigner = function(req, callback) { - if ( ! req.params.token || ! req.params.signer ) { - return callback(null, false); // no signer requested - } - - var layergroup_id = req.params.token; - var auth_token = req.params.auth_token; - - mapStore.load(layergroup_id, function(err, mapConfig) { - if (err) { - return callback(err); - } - - var authorized = templateMaps.isAuthorized(mapConfig.obj().template, auth_token); - - return callback(null, authorized); - }); - }; - - // Check if a request is authorized by api_key - // - // @param user - // @param req express request object - // @param callback function(err, authorized) - // NOTE: authorized is expected to be 0 or 1 (integer) - // - app.authorizedByAPIKey = function(user, req, callback) { - var givenKey = req.query.api_key || req.query.map_key; - if ( ! givenKey && req.body ) { - // check also in request body - givenKey = req.body.api_key || req.body.map_key; - } - if ( ! givenKey ) { - return callback(null, 0); // no api key, no authorization... - } - step( - function () { - metadataBackend.getUserMapKey(user, this); - }, - function checkApiKey(err, val){ - assert.ifError(err); - return val && givenKey == val; - }, - function finish(err, authorized) { - callback(err, authorized); - } - ); - }; - - /** - * Check access authorization - * - * @param req - standard req object. Importantly contains table and host information - * @param callback function(err, allowed) is access allowed not? - */ - app.authorize = function(req, callback) { - var self = this; - var user = req.context.user; - - step( - function () { - self.authorizedByAPIKey(user, req, this); - }, - function checkApiKey(err, authorized){ - if (req.profiler) { - req.profiler.done('authorizedByAPIKey'); - } - assert.ifError(err); - - // if not authorized by api_key, continue - if (!authorized) { - // not authorized by api_key, check if authorized by signer - return self.authorizedBySigner(req, this); - } - - // authorized by api key, login as the given username and stop - pgConnection.setDBAuth(user, req.params, function(err) { - callback(err, true); // authorized (or error) - }); - }, - function checkSignAuthorized(err, authorized) { - if (err) { - return callback(err); - } - - if ( ! authorized ) { - // request not authorized by signer. - - // if no signer name was given, let dbparams and - // PostgreSQL do the rest. - // - if ( ! req.params.signer ) { - return callback(null, true); // authorized so far - } - - // if signer name was given, return no authorization - return callback(null, false); - } - - pgConnection.setDBAuth(user, req.params, function(err) { - if (req.profiler) { - req.profiler.done('setDBAuth'); - } - callback(err, true); // authorized (or error) - }); - } - ); - }; - return app; }; From 96a6a0d980ccc9e83241859f9db74d035f0cbcda Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 15:05:45 +0200 Subject: [PATCH 07/57] Using MapStore, no need to attach it to app --- lib/cartodb/server.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index fdc028ce..3bfad338 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -132,7 +132,6 @@ module.exports = function(serverOptions) { pool: redisPool, expire_time: serverOptions.grainstore.default_layergroup_ttl }); - app.mapStore = mapStore; var onTileErrorStrategy; if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { @@ -521,14 +520,6 @@ module.exports = function(serverOptions) { function extractSQL(err) { assert.ifError(err); - // TODO: cached cache channel for token-based access should - // be constructed at renderer cache creation time - // See http://github.com/CartoDB/Windshaft-cartodb/issues/152 - if ( ! app.mapStore ) { - throw new Error('missing channel cache for token ' + req.params.token); - } - var mapStore = app.mapStore; - step( function loadFromStore() { mapStore.load(req.params.token, this); From 76823f7529c7b38d78da833729824088f62c2ee1 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 15:06:22 +0200 Subject: [PATCH 08/57] No need to pass a reference to itself --- lib/cartodb/server.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 3bfad338..27fd8fd9 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -257,8 +257,6 @@ module.exports = function(serverOptions) { ]; app.sendResponse = function(res, args) { - var that = this; - var statusCode; if ( res._windshaftStatusCode ) { // Added by our override of sendError @@ -294,7 +292,7 @@ module.exports = function(serverOptions) { } //console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + // mapCreateRoutes.join("\n")); - app.addCacheChannel(that, req, this); + app.addCacheChannel(req, this); }, function sendResponse(err/*, added*/) { if ( err ) console.log(err + err.stack); @@ -579,7 +577,7 @@ module.exports = function(serverOptions) { // @param cb function(err, channel) will be called when ready. // the channel parameter will be null if nothing was added // - app.addCacheChannel = function(app, req, cb) { + app.addCacheChannel = function(req, cb) { // skip non-GET requests, or requests for which there's no response if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } if (req.profiler) { From 5e2a20fbe03fe074301971c65c3df013bb8120bc Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 16:15:34 +0200 Subject: [PATCH 09/57] Tags layergroup instantiation with surrogate keys per affected tables --- .../cache/model/database_tables_entry.js | 20 ++++++++++++++ lib/cartodb/cache/surrogate_keys_cache.js | 14 +++++++++- lib/cartodb/controllers/map.js | 27 ++++++++++--------- 3 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 lib/cartodb/cache/model/database_tables_entry.js diff --git a/lib/cartodb/cache/model/database_tables_entry.js b/lib/cartodb/cache/model/database_tables_entry.js new file mode 100644 index 00000000..9bdb6e3d --- /dev/null +++ b/lib/cartodb/cache/model/database_tables_entry.js @@ -0,0 +1,20 @@ +var crypto = require('crypto'); + +function DatabaseTables(dbName, tableNames) { + this.namespace = 't'; + this.dbName = dbName; + this.tableNames = tableNames; +} + +module.exports = DatabaseTables; + + +DatabaseTables.prototype.key = function() { + return this.tableNames.map(function(tableName) { + return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName); + }.bind(this)); +}; + +function shortHashKey(target) { + return crypto.createHash('sha256').update(target).digest('base64').substring(0,6); +} diff --git a/lib/cartodb/cache/surrogate_keys_cache.js b/lib/cartodb/cache/surrogate_keys_cache.js index 77147917..237ac906 100644 --- a/lib/cartodb/cache/surrogate_keys_cache.js +++ b/lib/cartodb/cache/surrogate_keys_cache.js @@ -16,9 +16,21 @@ module.exports = SurrogateKeysCache; * @param cacheObject should respond to `key() -> String` method */ SurrogateKeysCache.prototype.tag = function(response, cacheObject) { - response.header('Surrogate-Key', cacheObject.key()); + var newKey = cacheObject.key(); + response.header('Surrogate-Key', appendSurrogateKey( + response.header('Surrogate-Key'), + Array.isArray(newKey) ? cacheObject.key().join(' ') : newKey + )); + }; +function appendSurrogateKey(currentKey, newKey) { + if (!!currentKey) { + newKey = currentKey + ' ' + newKey; + } + return newKey; +} + /** * @param cacheObject should respond to `key() -> String` method * @param {Function} callback diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index d5b43425..9e6fcf44 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -9,6 +9,7 @@ var MapConfig = windshaft.model.MapConfig; var Datasource = windshaft.model.Datasource; var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +var TablesCacheEntry = require('../cache/model/database_tables_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); @@ -147,13 +148,14 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, mapConfig, layergroup, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, this); }, function finish(err, layergroup) { if (err) { var statusCode = self.app.findStatusCode(err); self.app.sendError(res, { errors: [ err.message ] }, statusCode, 'ANONYMOUS LAYERGROUP', err); } else { + res.header('X-Layergroup-Id', layergroup.layergroupid); self.app.sendResponse(res, [layergroup, 200]); } } @@ -169,6 +171,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn var mapConfig; step( + function setupParams(){ + self.app.req2params(req, this); + }, function getTemplateParams() { prepareParamsFn(this); }, @@ -197,7 +202,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, mapConfig, layergroup, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -217,7 +222,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }; -MapController.prototype.afterLayergroupCreate = function(req, mapconfig, layergroup, callback) { +MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, callback) { var self = this; var username = req.context.user; @@ -276,16 +281,14 @@ MapController.prototype.afterLayergroupCreate = function(req, mapconfig, layergr layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString(); - var res = req.res; - if (res) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - res.header('Last-Modified', (new Date()).toUTCString()); - res.header('X-Cache-Channel', cacheChannel); + if (req.method === 'GET') { + var ttl = global.environment.varnish.layergroupTtl || 86400; + res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); + res.header('Last-Modified', (new Date()).toUTCString()); + res.header('X-Cache-Channel', cacheChannel); + if (result.affectedTables && result.affectedTables.length > 0) { + self.surrogateKeysCache.tag(res, new TablesCacheEntry(dbName, result.affectedTables)); } - - res.header('X-Layergroup-Id', layergroup.layergroupid); } return null; From d6447ef3112180d38567a164d0f3ee0010cb30f7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 16:36:41 +0200 Subject: [PATCH 10/57] Fix tests related to surrogate keys, includes tables --- test/acceptance/templates.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 79f24ac5..e0ba8bc7 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -3,7 +3,8 @@ var _ = require('underscore'); var redis = require('redis'); var step = require('step'); var strftime = require('strftime'); -var NamedMapsCacheEntry = require(__dirname + '/../../lib/cartodb/cache/model/named_maps_entry'); +var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry'); +var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry'); var redis_stats_db = 5; // Pollute the PG environment to make sure @@ -1543,7 +1544,11 @@ describe('template_api', function() { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); - helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()); + var expectedSurrogateKey = [ + new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(), + new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key() + ].join(' '); + helper.checkSurrogateKey(res, expectedSurrogateKey); return null; }, function finish(err) { @@ -1612,7 +1617,11 @@ describe('template_api', function() { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); - helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()); + var expectedSurrogateKey = [ + new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(), + new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key() + ].join(' '); + helper.checkSurrogateKey(res, expectedSurrogateKey); return null; }, function finish(err) { From 9355a5ca246d0e0bb1e6dff228b35df5fd2b13ba Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 16:54:08 +0200 Subject: [PATCH 11/57] Tests for surrogate keys in layergroup anonymous instantiation --- test/acceptance/multilayer.js | 41 +++++++++++++++++++++-------------- test/support/test_helper.js | 1 + 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index b23521ce..fb04eb07 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -17,6 +17,8 @@ var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); +var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry'); + ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) { var suiteName = 'multilayer:postgres=layergroup_url=' + layergroup_url; @@ -67,11 +69,8 @@ suite(suiteName, function() { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); assert.equal(parsedBody.last_updated, expected_last_updated); - if ( expected_token ) { - assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); - assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid); - } - else expected_token = parsedBody.layergroupid.split(':')[0]; + assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid); + expected_token = parsedBody.layergroupid.split(':')[0]; next(null, res); }); }, @@ -226,17 +225,23 @@ suite(suiteName, function() { test("get creation requests has cache", function(done) { - var layergroup = { - version: '1.0.0', - layers: [ - { options: { - sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' + - ' from test_table limit 2', - cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', - cartocss_version: '2.0.1' - } } - ] - }; + var layergroup = { + version: '1.0.0', + layers: [ + { options: { + sql: 'select cartodb_id, the_geom_webmercator from test_table', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.0.1', + interactivity: 'cartodb_id' + } }, + { options: { + sql: 'select cartodb_id, the_geom_webmercator from test_table_2', + cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', + cartocss_version: '2.0.2', + interactivity: 'cartodb_id' + } } + ] + }; var expected_token; step( @@ -255,6 +260,10 @@ suite(suiteName, function() { var parsedBody = JSON.parse(res.body); expected_token = parsedBody.layergroupid.split(':')[0]; helper.checkCache(res); + helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [ + 'public.test_table', + 'public.test_table_2' + ]).key().join(' ')); return null; }, function finish(err) { diff --git a/test/support/test_helper.js b/test/support/test_helper.js index f7924dad..00351eee 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -34,6 +34,7 @@ function lzma_compress_to_base64(payload, mode, callback) { // Throws on failure function checkNoCache(res) { assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); + assert.ok(!res.headers.hasOwnProperty('surrogate-key')); assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ? assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ? } From 36257f73b9da5f4dcf077e815d8edbfb43951d9f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jul 2015 17:18:50 +0200 Subject: [PATCH 12/57] Better format --- lib/cartodb/server.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 27fd8fd9..7ac8573b 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -455,7 +455,7 @@ module.exports = function(serverOptions) { function getPrivacy(){ authApi.authorize(req, this); }, - function gatekeep(err, authorized){ + function validateAuthorization(err, authorized) { if (req.profiler) { req.profiler.done('authorize'); } @@ -472,7 +472,9 @@ module.exports = function(serverOptions) { pgConnection.setDBConn(user, req.params, this); }, function finishSetup(err) { - if ( err ) { callback(err, req); return; } + if ( err ) { + return callback(err, req); + } // Add default database connection parameters // if none given From c295584864d7b3ca784bf5012fc4ef2c43e1a230 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 11:55:49 +0200 Subject: [PATCH 13/57] Cache channel now in layergroup controller Internal cache channel dbname+layergroupid cache must be unified in layergroup and map controllers Removes sendWithHeaders --- lib/cartodb/controllers/layergroup.js | 127 ++++++++++++- lib/cartodb/controllers/map.js | 10 +- lib/cartodb/controllers/named_maps.js | 2 +- lib/cartodb/server.js | 252 +++----------------------- 4 files changed, 157 insertions(+), 234 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b20b0ebc..68a23fe2 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -11,16 +11,21 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider' * @param {TileBackend} tileBackend * @param {PreviewBackend} previewBackend * @param {AttributesBackend} attributesBackend - * @param {{UserLimitsApi}} userLimitsApi + * @param {UserLimitsApi} userLimitsApi + * @param {QueryTablesApi} queryTablesApi * @constructor */ -function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi) { +function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi, + queryTablesApi) { this.app = app; this.mapStore = mapStore; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.attributesBackend = attributesBackend; this.userLimitsApi = userLimitsApi; + this.queryTablesApi = queryTablesApi; + + this.channelCache = {}; } module.exports = LayergroupController; @@ -62,7 +67,7 @@ LayergroupController.prototype.attributes = function(req, res) { var statusCode = self.app.findStatusCode(err); self.app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err); } else { - self.app.sendResponse(res, [tile, 200]); + self.sendResponse(req, res, [tile, 200]); } } ); @@ -149,7 +154,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); } else { - this.app.sendWithHeaders(res, tile, 200, headers); + this.sendResponse(req, res, [tile, headers, 200]); global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); } @@ -206,8 +211,120 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo self.app.sendError(res, {errors: ['' + err] }, self.app.findStatusCode(err), 'STATIC_MAP', err); } else { res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format); - self.app.sendResponse(res, [image, 200]); + self.sendResponse(req, res, [image, 200]); } } ); }; + +LayergroupController.prototype.sendResponse = function(req, res, args) { + var self = this; + + res.header('Cache-Control', 'public,max-age=31536000'); + + // Set Last-Modified header + var lastUpdated; + if (req.params.cache_buster) { + // Assuming cache_buster is a timestamp + lastUpdated = new Date(parseInt(req.params.cache_buster)); + } else { + lastUpdated = new Date(); + } + res.header('Last-Modified', lastUpdated.toUTCString()); + + step( + function getCacheChannel() { + self.cacheChannel(req, this); + }, + function sendResponse(err, cacheChannel) { + if (err) { + console.log('ERROR generating cache channel: ' + err); + } + if (!!cacheChannel) { + res.header('X-Cache-Channel', cacheChannel); + } + self.app.sendResponse(res, args); + } + ); + +}; + +LayergroupController.prototype.buildCacheChannel = function (dbName, tableNames){ + return dbName + ':' + tableNames.join(','); +}; + +LayergroupController.prototype.cacheChannel = function(req, callback) { + if (req.profiler) { + req.profiler.start('addCacheChannel'); + } + + var dbName = req.params.dbname; + + // no token means no tables associated + if (!req.params.token) { + return callback(null, this.buildCacheChannel(dbName, [])); + } + + var self = this; + + var cacheKey = [ dbName, req.params.token ].join(':'); + + step( + function checkCached() { + if ( self.channelCache.hasOwnProperty(cacheKey) ) { + return callback(null, self.channelCache[cacheKey]); + } + return null; + }, + function extractSQL(err) { + assert.ifError(err); + + step( + function loadFromStore() { + self.mapStore.load(req.params.token, this); + }, + function getSQL(err, mapConfig) { + if (req.profiler) { + req.profiler.done('mapStore_load'); + } + assert.ifError(err); + + var queries = mapConfig.getLayers() + .map(function(lyr) { + return lyr.options.sql; + }) + .filter(function(sql) { + return !!sql; + }); + + return queries.length ? queries.join(';') : null; + }, + this + ); + }, + function findAffectedTables(err, sql) { + assert.ifError(err); + + if ( ! sql ) { + throw new Error("this request doesn't need an X-Cache-Channel generated"); + } + + self.queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel + }, + function buildCacheChannel(err, tableNames) { + assert.ifError(err); + + if (req.profiler) { + req.profiler.done('affectedTables'); + } + + var cacheChannel = self.buildCacheChannel(dbName, tableNames); + self.channelCache[cacheKey] = cacheChannel; + + return cacheChannel; + }, + function finish(err, cacheChannel) { + callback(err, cacheChannel); + } + ); +}; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 9e6fcf44..bc558b8a 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -37,6 +37,8 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + + this.channelCache = {}; } module.exports = MapController; @@ -274,8 +276,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la req.profiler.done('queryTablesAndLastUpdated'); } assert.ifError(err); - var cacheChannel = self.app.buildCacheChannel(dbName, result.affectedTables); - self.app.channelCache[cacheKey] = cacheChannel; + var cacheChannel = self.buildCacheChannel(dbName, result.affectedTables); + self.channelCache[cacheKey] = cacheChannel; // last update for layergroup cache buster layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; @@ -298,3 +300,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la } ); }; + +MapController.prototype.buildCacheChannel = function (dbName, tableNames){ + return dbName + ':' + tableNames.join(','); +}; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 0e0ce057..d0245f8f 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -63,7 +63,7 @@ NamedMapsController.prototype.tile = function(req, res) { self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName())); res.setHeader('Content-Type', headers['Content-Type']); res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate'); - self.app.sendWithHeaders(res, tile, 200, headers); + self.app.sendResponse(res, [tile, headers, 200]); } } ); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7ac8573b..132db032 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -74,12 +74,6 @@ module.exports = function(serverOptions) { max_user_templates: global.environment.maxUserTemplates }); - // This is for Templated maps - // - // "named" is the official, "template" is for backward compatibility up to 1.6.x - // - var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)'; - var surrogateKeysCacheBackends = []; if (serverOptions.varnish_purge_enabled) { @@ -197,7 +191,8 @@ module.exports = function(serverOptions) { tileBackend, previewBackend, attributesBackend, - userLimitsApi + userLimitsApi, + queryTablesApi ).register(app); new controller.Map( @@ -243,98 +238,33 @@ module.exports = function(serverOptions) { } }); - // GET routes for which we don't want to request any caching. - // POST/PUT/DELETE requests are never cached anyway. - var noCacheGETRoutes = [ - '/', - '/version', - // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 - serverOptions.base_url_mapconfig, - serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', - template_baseurl, - template_baseurl + '/:template_id', - template_baseurl + '/:template_id/jsonp' - ]; - app.sendResponse = function(res, args) { - var statusCode; - if ( res._windshaftStatusCode ) { - // Added by our override of sendError - statusCode = res._windshaftStatusCode; - } else { - if ( args.length > 2 ) statusCode = args[2]; - else { - statusCode = args[1] || 200; + var req = res.req; + + if (global.environment && global.environment.api_hostname) { + res.header('X-Served-By-Host', global.environment.api_hostname); + } + + if (req && req.params && req.params.dbhost) { + res.header('X-Served-By-DB-Host', req.params.dbhost); + } + + if ( req && req.profiler ) { + res.header('X-Tiler-Profiler', req.profiler.toJSONString()); + } + +// res.send(body|status[, headers|status[, status]]) + res.send.apply(res, args); + + if ( req && req.profiler ) { + try { + // May throw due to dns, see + // See http://github.com/CartoDB/Windshaft/issues/166 + req.profiler.sendStats(); + } catch (err) { + console.error("error sending profiling stats: " + err); } } - var req = res.req; - step ( - function addCacheChannel() { - if ( ! req ) { - // having no associated request can happen when - // using fake response objects for testing layergroup - // creation - return false; - } - if ( ! req.params ) { - // service requests (/version, /) - // have no need for an X-Cache-Channel - return false; - } - if ( statusCode != 200 ) { - // We do not want to cache - // unsuccessful responses - return false; - } - if ( _.contains(noCacheGETRoutes, req.route.path) ) { -//console.log("Skipping cache channel in route:\n" + req.route.path); - return false; - } -//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + -// mapCreateRoutes.join("\n")); - app.addCacheChannel(req, this); - }, - function sendResponse(err/*, added*/) { - if ( err ) console.log(err + err.stack); - // When using custom results from tryFetch* methods, - // there is no "req" link in the result object. - // In those cases we don't want to send stats now - // as they will be sent at the real end of request - var req = res.req; - - if (global.environment && global.environment.api_hostname) { - res.header('X-Served-By-Host', global.environment.api_hostname); - } - - if (req && req.params && req.params.dbhost) { - res.header('X-Served-By-DB-Host', req.params.dbhost); - } - - if ( req && req.profiler ) { - res.header('X-Tiler-Profiler', req.profiler.toJSONString()); - } - - res.send.apply(res, args); - - if ( req && req.profiler ) { - try { - // May throw due to dns, see - // See http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - console.error("error sending profiling stats: " + err); - } - } - return null; - }, - function finish(err) { - if ( err ) console.log(err + err.stack); - } - ); - }; - - app.sendWithHeaders = function(res, what, status, headers) { - app.sendResponse(res, [what, headers, status]); }; app.sendError = function(res, err, statusCode, label, tolog) { @@ -490,136 +420,6 @@ module.exports = function(serverOptions) { ); }; - // TODO: review lifetime of elements of this cache - // NOTE: by-token indices should only be dropped when - // the corresponding layegroup is dropped, because - // we have no SQL after layer creation. - app.channelCache = {}; - - app.buildCacheChannel = function (dbName, tableNames){ - return dbName + ':' + tableNames.join(','); - }; - - app.generateCacheChannel = function(app, req, callback){ - // Build channelCache key - var dbName = req.params.dbname; - var cacheKey = [ dbName, req.params.token ].join(':'); - - // no token means no tables associated - if (!req.params.token) { - return callback(null, this.buildCacheChannel(dbName, [])); - } - - step( - function checkCached() { - if ( app.channelCache.hasOwnProperty(cacheKey) ) { - return callback(null, app.channelCache[cacheKey]); - } - return null; - }, - function extractSQL(err) { - assert.ifError(err); - - step( - function loadFromStore() { - mapStore.load(req.params.token, this); - }, - function getSQL(err, mapConfig) { - if (req.profiler) { - req.profiler.done('mapStore_load'); - } - assert.ifError(err); - - var queries = mapConfig.getLayers() - .map(function(lyr) { - return lyr.options.sql; - }) - .filter(function(sql) { - return !!sql; - }); - - return queries.length ? queries.join(';') : null; - }, - this - ); - }, - function findAffectedTables(err, sql) { - assert.ifError(err); - - if ( ! sql ) { - throw new Error("this request doesn't need an X-Cache-Channel generated"); - } - - queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel - }, - function buildCacheChannel(err, tableNames) { - assert.ifError(err); - - if (req.profiler) { - req.profiler.done('affectedTables'); - } - - var cacheChannel = app.buildCacheChannel(dbName,tableNames); - app.channelCache[cacheKey] = cacheChannel; - - return cacheChannel; - }, - function finish(err, cacheChannel) { - callback(err, cacheChannel); - } - ); - }; - - // Set the cache chanel info to invalidate the cache on the frontend server - // - // @param req The request object. - // The function will have no effect unless req.res exists. - // It is expected that req.params contains 'table' and 'dbname' - // - // @param cb function(err, channel) will be called when ready. - // the channel parameter will be null if nothing was added - // - app.addCacheChannel = function(req, cb) { - // skip non-GET requests, or requests for which there's no response - if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } - if (req.profiler) { - req.profiler.start('addCacheChannel'); - } - var res = req.res; - if ( req.params.token ) { - res.header('Cache-Control', 'public,max-age=31536000'); // 1 year - } else { - var ttl = global.environment.varnish.ttl || 86400; - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); - } - - // Set Last-Modified header - var lastUpdated; - if ( req.params.cache_buster ) { - // Assuming cache_buster is a timestamp - // FIXME: store lastModified in the cache channel instead - lastUpdated = new Date(parseInt(req.params.cache_buster)); - } else { - lastUpdated = new Date(); - } - res.header('Last-Modified', lastUpdated.toUTCString()); - - app.generateCacheChannel(app, req, function(err, channel){ - if (req.profiler) { - req.profiler.done('generateCacheChannel'); - req.profiler.end(); - } - if ( ! err ) { - res.header('X-Cache-Channel', channel); - cb(null, channel); - } else { - console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err )); - // TODO: evaluate if we should bubble up the error instead - cb(null, 'ERROR'); - } - }); - }; - return app; }; From e8b58451746bb5f2fc0c083e7e6215774c8eec0f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 13:40:41 +0200 Subject: [PATCH 14/57] Shared cache for affected tables in layergroup and map controllers --- lib/cartodb/api/query_tables_api.js | 31 ++++++++- .../cache/layergroup_affected_tables.js | 22 +++++++ .../cache/model/database_tables_entry.js | 4 ++ lib/cartodb/controllers/layergroup.js | 65 +++++++------------ lib/cartodb/controllers/map.js | 42 +++++++----- lib/cartodb/server.js | 11 +++- test/acceptance/multilayer_server.js | 4 +- test/acceptance/templates.js | 4 +- 8 files changed, 120 insertions(+), 63 deletions(-) create mode 100644 lib/cartodb/cache/layergroup_affected_tables.js diff --git a/lib/cartodb/api/query_tables_api.js b/lib/cartodb/api/query_tables_api.js index fc0e2932..576f165f 100644 --- a/lib/cartodb/api/query_tables_api.js +++ b/lib/cartodb/api/query_tables_api.js @@ -48,7 +48,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { if (err || rows.length === 0) { var msg = err.message ? err.message : err; - callback(new Error('could not fetch affected tables and last updated time: ' + msg)); + callback(new Error('could not fetch affected tables or last updated time: ' + msg)); return; } @@ -65,6 +65,35 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { }); } +QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) { + if (!Array.isArray(tableNames) || tableNames.length === 0) { + return callback(null, 0); + } + + var query = [ + 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max', + 'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[', + tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','), + '])' + ].join(' '); + + this.pgQueryRunner.run(username, query, handleLastUpdatedTimeRows, callback); +}; + +function handleLastUpdatedTimeRows(err, rows, callback) { + if (err) { + var msg = err.message ? err.message : err; + return callback(new Error('could not fetch affected tables or last updated time: ' + msg)); + } + // when the table has not updated_at means it hasn't been changed so a default last_updated is set + var lastUpdated = 0; + if (rows.length !== 0) { + lastUpdated = rows[0].max || 0; + } + + return callback(null, lastUpdated*1000); +} + function prepareSql(sql) { return sql .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') diff --git a/lib/cartodb/cache/layergroup_affected_tables.js b/lib/cartodb/cache/layergroup_affected_tables.js new file mode 100644 index 00000000..c2cccc71 --- /dev/null +++ b/lib/cartodb/cache/layergroup_affected_tables.js @@ -0,0 +1,22 @@ +function LayergroupAffectedTables() { + // layergroupId -> affected tables cache + this.cache = {}; +} + +module.exports = LayergroupAffectedTables; + +LayergroupAffectedTables.prototype.hasAffectedTables = function(dbName, layergroupId) { + return this.cache.hasOwnProperty(createKey(dbName, layergroupId)); +}; + +LayergroupAffectedTables.prototype.set = function(dbName, layergroupId, affectedTables) { + this.cache[createKey(dbName, layergroupId)] = affectedTables; +}; + +LayergroupAffectedTables.prototype.get = function(dbName, layergroupId) { + return this.cache[createKey(dbName, layergroupId)]; +}; + +function createKey(dbName, layergroupId) { + return dbName + ':' + layergroupId; +} diff --git a/lib/cartodb/cache/model/database_tables_entry.js b/lib/cartodb/cache/model/database_tables_entry.js index 9bdb6e3d..4d269137 100644 --- a/lib/cartodb/cache/model/database_tables_entry.js +++ b/lib/cartodb/cache/model/database_tables_entry.js @@ -15,6 +15,10 @@ DatabaseTables.prototype.key = function() { }.bind(this)); }; +DatabaseTables.prototype.getCacheChannel = function() { + return this.dbName + ':' + this.tableNames.join(','); +}; + function shortHashKey(target) { return crypto.createHash('sha256').update(target).digest('base64').substring(0,6); } diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 68a23fe2..0522e3b5 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -4,6 +4,7 @@ var step = require('step'); var cors = require('../middleware/cors'); var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); +var TablesCacheEntry = require('../cache/model/database_tables_entry'); /** * @param app @@ -11,19 +12,23 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider' * @param {TileBackend} tileBackend * @param {PreviewBackend} previewBackend * @param {AttributesBackend} attributesBackend + * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {QueryTablesApi} queryTablesApi + * @param {LayergroupAffectedTables} layergroupAffectedTables * @constructor */ -function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi, - queryTablesApi) { +function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, surrogateKeysCache, + userLimitsApi, queryTablesApi, layergroupAffectedTables) { this.app = app; this.mapStore = mapStore; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.attributesBackend = attributesBackend; + this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.queryTablesApi = queryTablesApi; + this.layergroupAffectedTables = layergroupAffectedTables; this.channelCache = {}; } @@ -232,16 +237,19 @@ LayergroupController.prototype.sendResponse = function(req, res, args) { } res.header('Last-Modified', lastUpdated.toUTCString()); + var dbName = req.params.dbname; step( - function getCacheChannel() { - self.cacheChannel(req, this); + function getAffectedTables() { + self.getAffectedTables(req.context.user, dbName, req.params.token, this); }, - function sendResponse(err, cacheChannel) { + function sendResponse(err, affectedTables) { if (err) { console.log('ERROR generating cache channel: ' + err); } - if (!!cacheChannel) { - res.header('X-Cache-Channel', cacheChannel); + if (!!affectedTables) { + var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables); + res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); + self.surrogateKeysCache.tag(res, tablesCacheEntry); } self.app.sendResponse(res, args); } @@ -249,30 +257,13 @@ LayergroupController.prototype.sendResponse = function(req, res, args) { }; -LayergroupController.prototype.buildCacheChannel = function (dbName, tableNames){ - return dbName + ':' + tableNames.join(','); -}; - -LayergroupController.prototype.cacheChannel = function(req, callback) { - if (req.profiler) { - req.profiler.start('addCacheChannel'); - } - - var dbName = req.params.dbname; - - // no token means no tables associated - if (!req.params.token) { - return callback(null, this.buildCacheChannel(dbName, [])); - } - +LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) { var self = this; - var cacheKey = [ dbName, req.params.token ].join(':'); - step( function checkCached() { - if ( self.channelCache.hasOwnProperty(cacheKey) ) { - return callback(null, self.channelCache[cacheKey]); + if (self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) { + return callback(null, self.layergroupAffectedTables.get(dbName, layergroupId)); } return null; }, @@ -281,12 +272,9 @@ LayergroupController.prototype.cacheChannel = function(req, callback) { step( function loadFromStore() { - self.mapStore.load(req.params.token, this); + self.mapStore.load(layergroupId, this); }, function getSQL(err, mapConfig) { - if (req.profiler) { - req.profiler.done('mapStore_load'); - } assert.ifError(err); var queries = mapConfig.getLayers() @@ -309,22 +297,17 @@ LayergroupController.prototype.cacheChannel = function(req, callback) { throw new Error("this request doesn't need an X-Cache-Channel generated"); } - self.queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel + self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel }, function buildCacheChannel(err, tableNames) { assert.ifError(err); - if (req.profiler) { - req.profiler.done('affectedTables'); - } + self.layergroupAffectedTables.set(dbName, layergroupId, tableNames); - var cacheChannel = self.buildCacheChannel(dbName, tableNames); - self.channelCache[cacheKey] = cacheChannel; - - return cacheChannel; + return tableNames; }, - function finish(err, cacheChannel) { - callback(err, cacheChannel); + function finish(err, affectedTables) { + callback(err, affectedTables); } ); }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index bc558b8a..1daa4ac3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -23,11 +23,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye * @param metadataBackend * @param {QueryTablesApi} queryTablesApi * @param {SurrogateKeysCache} surrogateKeysCache - * @param {{UserLimitsApi}} userLimitsApi + * @param {UserLimitsApi} userLimitsApi + * @param {LayergroupAffectedTables} layergroupAffectedTables * @constructor */ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi, - surrogateKeysCache, userLimitsApi) { + surrogateKeysCache, userLimitsApi, layergroupAffectedTables) { this.app = app; this.pgConnection = pgConnection; this.templateMaps = templateMaps; @@ -36,9 +37,9 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack this.queryTablesApi = queryTablesApi; this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; - this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + this.layergroupAffectedTables = layergroupAffectedTables; - this.channelCache = {}; + this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); } module.exports = MapController; @@ -265,31 +266,46 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la }).join(';'); var dbName = req.params.dbname; - var cacheKey = dbName + ':' + layergroup.layergroupid; + var layergroupId = layergroup.layergroupid; step( - function getAffectedTablesAndLastUpdatedTime() { - self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); + function checkCachedAffectedTables() { + return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId); + }, + function getAffectedTablesAndLastUpdatedTime(err, hasCache) { + assert.ifError(err); + if (hasCache) { + var next = this; + var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId); + self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) { + if (err) { + return next(err); + } + return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime }); + }); + } else { + self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); + } }, function handleAffectedTablesAndLastUpdatedTime(err, result) { if (req.profiler) { req.profiler.done('queryTablesAndLastUpdated'); } assert.ifError(err); - var cacheChannel = self.buildCacheChannel(dbName, result.affectedTables); - self.channelCache[cacheKey] = cacheChannel; + self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables); // last update for layergroup cache buster layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString(); if (req.method === 'GET') { + var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); var ttl = global.environment.varnish.layergroupTtl || 86400; res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); res.header('Last-Modified', (new Date()).toUTCString()); - res.header('X-Cache-Channel', cacheChannel); + res.header('X-Cache-Channel', tableCacheEntry.getCacheChannel()); if (result.affectedTables && result.affectedTables.length > 0) { - self.surrogateKeysCache.tag(res, new TablesCacheEntry(dbName, result.affectedTables)); + self.surrogateKeysCache.tag(res, tableCacheEntry); } } @@ -300,7 +316,3 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la } ); }; - -MapController.prototype.buildCacheChannel = function (dbName, tableNames){ - return dbName + ':' + tableNames.join(','); -}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 132db032..e01d99b1 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -20,6 +20,7 @@ var TemplateMaps = require('./backends/template_maps.js'); var QueryTablesApi = require('./api/query_tables_api'); var UserLimitsApi = require('./api/user_limits_api'); var AuthApi = require('./api/auth_api'); +var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); var PgQueryRunner = require('./backends/pg_query_runner'); var PgConnection = require('./backends/pg_connection'); @@ -161,6 +162,9 @@ module.exports = function(serverOptions) { var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); + app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); app.findStatusCode = function(err) { @@ -191,8 +195,10 @@ module.exports = function(serverOptions) { tileBackend, previewBackend, attributesBackend, + surrogateKeysCache, userLimitsApi, - queryTablesApi + queryTablesApi, + layergroupAffectedTablesCache ).register(app); new controller.Map( @@ -203,7 +209,8 @@ module.exports = function(serverOptions) { metadataBackend, queryTablesApi, surrogateKeysCache, - userLimitsApi + userLimitsApi, + layergroupAffectedTablesCache ).register(app); new controller.NamedMaps( diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index 4845545d..463d41d6 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -318,7 +318,7 @@ describe('tests from old api translated to multilayer', function() { var parsed = JSON.parse(res.body); assert.deepEqual(parsed, { - errors: ["Error: could not fetch affected tables and last updated time: fake error message"] + errors: ["Error: could not fetch affected tables or last updated time: fake error message"] }); done(); @@ -346,7 +346,7 @@ describe('tests from old api translated to multilayer', function() { }; // reset internal cacheChannel cache - server.channelCache = {}; + server.layergroupAffectedTablesCache.cache = {}; assert.response(server, { diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index e0ba8bc7..1f3e4df5 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -21,7 +21,7 @@ var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); describe('template_api', function() { - server.channelCache = {}; + server.layergroupAffectedTablesCache.cache = {}; var redis_client = redis.createClient(global.environment.redis.port); @@ -1155,7 +1155,7 @@ describe('template_api', function() { assert.ok(cc); assert.ok(cc.match, /ciao/, cc); // hack simulating restart... - server.channelCache = {}; // need to clean channel cache + server.layergroupAffectedTablesCache.cache = {}; // need to clean channel cache var get_request = { url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1', method: 'GET', From c97610ad59a9df5ce37fb8745d6690708b184c64 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 14:30:37 +0200 Subject: [PATCH 15/57] style --- lib/cartodb/models/mapconfig/named_map_provider.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index 3ee5b18e..40c8bfe4 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -153,7 +153,7 @@ NamedMapMapConfigProvider.prototype.filter = function(key) { }; // Configure bases for cache keys suitable for string interpolation -var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}'; +var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}'; var rendererKey = baseKey + ':{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}'; var baseKeyTpl = dot.template(baseKey); From a0d86ac5dc5786d9d629b1cb28482607e69eef75 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 16:28:51 +0200 Subject: [PATCH 16/57] Update news --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index b7a68599..51f054a5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,9 @@ Released 2015-mm-dd +Enhancements: + - Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names + ## 2.7.1 From 4f84138ade22547bc43a86d8b721d3cdd331e7b9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 16:32:04 +0200 Subject: [PATCH 17/57] Release 2.7.2 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 51f054a5..b24861b5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## 2.7.2 -Released 2015-mm-dd +Released 2015-07-14 Enhancements: - Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names From 436c334f5a3cdb79d04456d718c2f4d0b54ac245 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 16:33:12 +0200 Subject: [PATCH 18/57] Stubs next version --- NEWS.md | 5 +++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index b24861b5..fd371c71 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 2.7.3 + +Released 2015-mm-dd + + ## 2.7.2 Released 2015-07-14 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 3785e62e..629e1dd2 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.7.2", + "version": "2.7.3", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index e6432fa9..e4cb2b75 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.7.2", + "version": "2.7.3", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 8e8f618a22ce542c7a267a4d2394bb076d2a6376 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 17:33:42 +0200 Subject: [PATCH 19/57] assert instead of ifs --- lib/cartodb/backends/pg_connection.js | 7 ++++--- lib/cartodb/backends/pg_query_runner.js | 9 +++------ lib/cartodb/backends/template_maps.js | 25 ++++++++----------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/lib/cartodb/backends/pg_connection.js b/lib/cartodb/backends/pg_connection.js index 7e4aad15..6cc432fb 100644 --- a/lib/cartodb/backends/pg_connection.js +++ b/lib/cartodb/backends/pg_connection.js @@ -1,3 +1,4 @@ +var assert = require('assert'); var step = require('step'); var _ = require('underscore'); @@ -29,7 +30,7 @@ PgConnection.prototype.setDBAuth = function(username, params, callback) { self.metadataBackend.getUserId(username, this); }, function(err, user_id) { - if (err) throw err; + assert.ifError(err); user_params.user_id = user_id; var dbuser = _.template(auth_user, user_params); _.extend(params, {dbuser:dbuser}); @@ -41,7 +42,7 @@ PgConnection.prototype.setDBAuth = function(username, params, callback) { self.metadataBackend.getUserDBPass(username, this); }, function(err, user_password) { - if (err) throw err; + assert.ifError(err); user_params.user_password = user_password; if ( auth_pass ) { var dbpass = _.template(auth_pass, user_params); @@ -81,7 +82,7 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) { self.metadataBackend.getUserDBConnectionParams(dbowner, this); }, function extendParams(err, dbParams){ - if (err) throw err; + assert.ifError(err); // we don't want null values or overwrite a non public user if (params.dbuser != 'publicuser' || !dbParams.dbuser) { delete dbParams.dbuser; diff --git a/lib/cartodb/backends/pg_query_runner.js b/lib/cartodb/backends/pg_query_runner.js index ed6c88db..0ba40c3e 100644 --- a/lib/cartodb/backends/pg_query_runner.js +++ b/lib/cartodb/backends/pg_query_runner.js @@ -1,3 +1,4 @@ +var assert = require('assert'); var PSQL = require('cartodb-psql'); var step = require('step'); @@ -18,15 +19,11 @@ PgQueryRunner.prototype.run = function(username, query, queryHandler, callback) self.pgConnection.setDBAuth(username, params, this); }, function setConn(err) { - if (err) { - throw err; - } + assert.ifError(err); self.pgConnection.setDBConn(username, params, this); }, function executeQuery(err) { - if (err) { - throw err; - } + assert.ifError(err); var psql = new PSQL({ user: params.dbuser, pass: params.dbpass, diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index d026fa7e..544bdf1f 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -1,3 +1,4 @@ +var assert = require('assert'); var crypto = require('crypto'); var step = require('step'); var _ = require('underscore'); @@ -76,7 +77,7 @@ o._redisCmd = function(redisFunc, redisArgs, callback) { that.redis_pool.acquire(db, this); }, function executeQuery(err, data) { - if ( err ) throw err; + assert.ifError(err); redisClient = data; redisArgs.push(this); redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs); @@ -214,9 +215,7 @@ o.addTemplate = function(owner, template, callback) { self._redisCmd('HLEN', [ userTemplatesKey ], this); }, function installTemplateIfDoesNotExist(err, numberOfTemplates) { - if ( err ) { - throw err; - } + assert.ifError(err); if ( limit && numberOfTemplates >= limit ) { throw new Error("User '" + owner + "' reached limit on number of templates " + "("+ numberOfTemplates + "/" + limit + ")"); @@ -224,9 +223,7 @@ o.addTemplate = function(owner, template, callback) { self._redisCmd('HSETNX', [ userTemplatesKey, templateName, JSON.stringify(template) ], this); }, function validateInstallation(err, wasSet) { - if ( err ) { - throw err; - } + assert.ifError(err); if ( ! wasSet ) { throw new Error("Template '" + templateName + "' of user '" + owner + "' already exists"); } @@ -259,9 +256,7 @@ o.delTemplate = function(owner, tpl_id, callback) { self._redisCmd('HDEL', [ self.key_usr_tpl({ owner:owner }), tpl_id ], this); }, function handleDeletion(err, deleted) { - if (err) { - throw err; - } + assert.ifError(err); if (!deleted) { throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); } @@ -317,18 +312,14 @@ o.updTemplate = function(owner, tpl_id, template, callback) { self._redisCmd('HGET', [ userTemplatesKey, tpl_id ], this); }, function updateTemplate(err, currentTemplate) { - if (err) { - throw err; - } + assert.ifError(err); if (!currentTemplate) { throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); } self._redisCmd('HSET', [ userTemplatesKey, templateName, JSON.stringify(template) ], this); }, function handleTemplateUpdate(err, didSetNewField) { - if (err) { - throw err; - } + assert.ifError(err); if (didSetNewField) { console.warn('New template created on update operation'); } @@ -372,7 +363,7 @@ o.getTemplate = function(owner, tpl_id, callback) { self._redisCmd('HGET', [ self.key_usr_tpl({owner:owner}), tpl_id ], this); }, function parseTemplate(err, tpl_val) { - if ( err ) throw err; + assert.ifError(err); return JSON.parse(tpl_val); }, function finish(err, tpl) { From 7247b20686e48057c0c1d854babf131d2ddd6540 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 17:34:05 +0200 Subject: [PATCH 20/57] Unify sendResponse in named maps controller --- lib/cartodb/controllers/named_maps.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index d0245f8f..2aa1a4ee 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -27,6 +27,13 @@ NamedMapsController.prototype.register = function(app) { ); }; +NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) { + this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName())); + res.setHeader('Content-Type', headers['Content-Type']); + res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate'); + this.app.sendResponse(res, [resource, 200]); +}; + NamedMapsController.prototype.tile = function(req, res) { var self = this; @@ -60,10 +67,7 @@ NamedMapsController.prototype.tile = function(req, res) { } self.app.sendError(res, err, self.app.findStatusCode(err), 'NAMED_MAP_TILE', err); } else { - self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName())); - res.setHeader('Content-Type', headers['Content-Type']); - res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate'); - self.app.sendResponse(res, [tile, headers, 200]); + self.sendResponse(req, res, tile, headers, namedMapProvider); } } ); @@ -158,10 +162,7 @@ NamedMapsController.prototype.staticMap = function(req, res) { } self.app.sendError(res, err, self.app.findStatusCode(err), 'STATIC_VIZ_MAP', err); } else { - self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName())); - res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format); - res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate'); - self.app.sendResponse(res, [image, 200]); + self.sendResponse(req, res, image, headers, namedMapProvider); } } ); From 4aabe9d946b5e6e161f0b8dafbac242b0116bbaf Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 20:10:55 +0200 Subject: [PATCH 21/57] Named maps controller adding cache headers This requires a cache for affected tables as it is hitting db for every request right now --- lib/cartodb/controllers/named_maps.js | 41 +++++++++++++++++-- .../models/mapconfig/named_map_provider.js | 30 ++++++++++++++ lib/cartodb/server.js | 3 +- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 2aa1a4ee..9b27ad4b 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -5,9 +5,10 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var cors = require('../middleware/cors'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); +var TablesCacheEntry = require('../cache/model/database_tables_entry'); function NamedMapsController(app, pgConnection, templateMaps, tileBackend, previewBackend, surrogateKeysCache, - tablesExtentApi, userLimitsApi) { + tablesExtentApi, userLimitsApi, queryTablesApi) { this.app = app; this.pgConnection = pgConnection; this.templateMaps = templateMaps; @@ -16,6 +17,7 @@ function NamedMapsController(app, pgConnection, templateMaps, tileBackend, previ this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; this.userLimitsApi = userLimitsApi; + this.queryTablesApi = queryTablesApi; } module.exports = NamedMapsController; @@ -29,9 +31,40 @@ NamedMapsController.prototype.register = function(app) { NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) { this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName())); - res.setHeader('Content-Type', headers['Content-Type']); - res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate'); - this.app.sendResponse(res, [resource, 200]); + res.header('Content-Type', headers['Content-Type']); + res.header('Cache-Control', 'public,max-age=7200,must-revalidate'); + + var self = this; + + var dbName = req.params.dbname; + step( + function getAffectedTablesAndLastUpdatedTime() { + namedMapProvider.getAffectedTablesAndLastUpdatedTime(self.queryTablesApi, this); + }, + function sendResponse(err, result) { + req.profiler.done('affectedTables'); + if (err) { + console.log('ERROR generating cache channel: ' + err); + } + if (!result || !!result.affectedTables) { + // we increase cache control as we can invalidate it + res.header('Cache-Control', 'public,max-age=31536000'); + + var lastModifiedDate; + if (Number.isFinite(result.lastUpdatedTime)) { + lastModifiedDate = new Date(result.lastUpdatedTime); + } else { + lastModifiedDate = new Date(); + } + res.header('Last-Modified', lastModifiedDate.toUTCString()); + + var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); + res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); + self.surrogateKeysCache.tag(res, tablesCacheEntry); + } + self.app.sendResponse(res, [resource, 200]); + } + ); }; NamedMapsController.prototype.tile = function(req, res) { diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index 40c8bfe4..c9970cef 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -24,6 +24,8 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, ow // use template after call to mapConfig this.template = null; + this.affectedTablesAndLastUpdate = null; + // providing this.err = null; this.mapConfig = null; @@ -198,3 +200,31 @@ NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, call NamedMapMapConfigProvider.prototype.getTemplateName = function() { return this.templateName; }; + +NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(queryTablesApi, callback) { + var self = this; + + if (this.affectedTablesAndLastUpdate !== null) { + return this.affectedTablesAndLastUpdate; + } + + step( + function getMapConfig() { + self.getMapConfig(this); + }, + function getSql(err, mapConfig) { + assert.ifError(err); + return mapConfig.getLayers().map(function(layer) { + return layer.options.sql; + }).join(';'); + }, + function getAffectedTables(err, sql) { + assert.ifError(err); + queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this); + }, + function finish(err, result) { + self.affectedTablesAndLastUpdate = result; + return callback(err, result); + } + ); +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index e01d99b1..11d88361 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -221,7 +221,8 @@ module.exports = function(serverOptions) { previewBackend, surrogateKeysCache, tablesExtentApi, - userLimitsApi + userLimitsApi, + queryTablesApi ).register(app); new controller.NamedMapsAdmin(app, templateMaps, authApi).register(app); From 6d3ef11a7c5ad9c8be46bbb6e1a0fc1d26b52929 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 20:11:49 +0200 Subject: [PATCH 22/57] Fix cache usage in layergroup affected tables --- lib/cartodb/controllers/layergroup.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 0522e3b5..0d932008 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -243,6 +243,7 @@ LayergroupController.prototype.sendResponse = function(req, res, args) { self.getAffectedTables(req.context.user, dbName, req.params.token, this); }, function sendResponse(err, affectedTables) { + req.profiler.done('affectedTables'); if (err) { console.log('ERROR generating cache channel: ' + err); } @@ -258,18 +259,14 @@ LayergroupController.prototype.sendResponse = function(req, res, args) { }; LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) { + + if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) { + return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId)); + } + var self = this; - step( - function checkCached() { - if (self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) { - return callback(null, self.layergroupAffectedTables.get(dbName, layergroupId)); - } - return null; - }, - function extractSQL(err) { - assert.ifError(err); - + function extractSQL() { step( function loadFromStore() { self.mapStore.load(layergroupId, this); From 07c920bad5d3e23c674afac491880adc6ca69af6 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 20:53:06 +0200 Subject: [PATCH 23/57] Use named map provider cache to retrieve providers --- lib/cartodb/cache/named_map_provider_cache.js | 24 +++++++++++++++++++ lib/cartodb/controllers/map.js | 1 + lib/cartodb/controllers/named_maps.js | 22 +++++------------ .../models/mapconfig/named_map_provider.js | 9 +++---- lib/cartodb/server.js | 10 ++++---- 5 files changed, 41 insertions(+), 25 deletions(-) create mode 100644 lib/cartodb/cache/named_map_provider_cache.js diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js new file mode 100644 index 00000000..3acde0e3 --- /dev/null +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -0,0 +1,24 @@ +var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); + +function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) { + this.templateMaps = templateMaps; + this.pgConnection = pgConnection; + this.userLimitsApi = userLimitsApi; + this.queryTablesApi = queryTablesApi; +} + +module.exports = NamedMapProviderCache; + +NamedMapProviderCache.prototype.get = function(user, templateId, config, authToken, params) { + return new NamedMapMapConfigProvider( + this.templateMaps, + this.pgConnection, + this.userLimitsApi, + this.queryTablesApi, + user, + templateId, + config, + authToken, + params + ); +}; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 1daa4ac3..45f17797 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -186,6 +186,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn self.templateMaps, self.pgConnection, self.userLimitsApi, + self.queryTablesApi, cdbuser, req.params.template_id, templateParams, diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 9b27ad4b..a0be85b7 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -4,20 +4,16 @@ var _ = require('underscore'); var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var cors = require('../middleware/cors'); -var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var TablesCacheEntry = require('../cache/model/database_tables_entry'); -function NamedMapsController(app, pgConnection, templateMaps, tileBackend, previewBackend, surrogateKeysCache, - tablesExtentApi, userLimitsApi, queryTablesApi) { +function NamedMapsController(app, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, + tablesExtentApi) { this.app = app; - this.pgConnection = pgConnection; - this.templateMaps = templateMaps; + this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; - this.userLimitsApi = userLimitsApi; - this.queryTablesApi = queryTablesApi; } module.exports = NamedMapsController; @@ -39,7 +35,7 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header var dbName = req.params.dbname; step( function getAffectedTablesAndLastUpdatedTime() { - namedMapProvider.getAffectedTablesAndLastUpdatedTime(self.queryTablesApi, this); + namedMapProvider.getAffectedTablesAndLastUpdatedTime(this); }, function sendResponse(err, result) { req.profiler.done('affectedTables'); @@ -78,10 +74,7 @@ NamedMapsController.prototype.tile = function(req, res) { self.app.req2params(req, this); }, function getTile() { - namedMapProvider = new NamedMapMapConfigProvider( - self.templateMaps, - self.pgConnection, - self.userLimitsApi, + namedMapProvider = self.namedMapProviderCache.get( cdbUser, req.params.template_id, req.query.config, @@ -122,10 +115,7 @@ NamedMapsController.prototype.staticMap = function(req, res) { }, function getTemplate(err) { assert.ifError(err); - namedMapProvider = new NamedMapMapConfigProvider( - self.templateMaps, - self.pgConnection, - self.userLimitsApi, + namedMapProvider = self.namedMapProviderCache.get( cdbUser, req.params.template_id, req.query.config, diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index c9970cef..f84e5bbf 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -10,11 +10,12 @@ var templateName = require('../../backends/template_maps').templateName; * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, owner, templateId, config, authToken, - params) { +function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, + owner, templateId, config, authToken, params) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.userLimitsApi = userLimitsApi; + this.queryTablesApi = queryTablesApi; this.owner = owner; this.templateName = templateName(templateId); this.config = config; @@ -201,7 +202,7 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() { return this.templateName; }; -NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(queryTablesApi, callback) { +NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) { var self = this; if (this.affectedTablesAndLastUpdate !== null) { @@ -220,7 +221,7 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi }, function getAffectedTables(err, sql) { assert.ifError(err); - queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this); + self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this); }, function finish(err, result) { self.affectedTablesAndLastUpdate = result; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 11d88361..ebb1affd 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -21,6 +21,7 @@ var QueryTablesApi = require('./api/query_tables_api'); var UserLimitsApi = require('./api/user_limits_api'); var AuthApi = require('./api/auth_api'); var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); +var NamedMapProviderCache = require('./cache/named_map_provider_cache'); var PgQueryRunner = require('./backends/pg_query_runner'); var PgConnection = require('./backends/pg_connection'); @@ -165,6 +166,8 @@ module.exports = function(serverOptions) { var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi); + var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); app.findStatusCode = function(err) { @@ -215,14 +218,11 @@ module.exports = function(serverOptions) { new controller.NamedMaps( app, - pgConnection, - templateMaps, + namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, - tablesExtentApi, - userLimitsApi, - queryTablesApi + tablesExtentApi ).register(app); new controller.NamedMapsAdmin(app, templateMaps, authApi).register(app); From 722705468fa140cd4211ed17ed45f29da2a8dcf8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 20:53:26 +0200 Subject: [PATCH 24/57] Not adding surrogate keys for empty affected tables --- lib/cartodb/controllers/named_maps.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index a0be85b7..6936332f 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -56,7 +56,9 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); - self.surrogateKeysCache.tag(res, tablesCacheEntry); + if (result.affectedTables.length > 0) { + self.surrogateKeysCache.tag(res, tablesCacheEntry); + } } self.app.sendResponse(res, [resource, 200]); } From 91ab64dda981540e4d27eec7054666ba2a840481 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 21:00:27 +0200 Subject: [PATCH 25/57] Fix cached result in getAffectedTablesAndLastUpdatedTime --- lib/cartodb/models/mapconfig/named_map_provider.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index f84e5bbf..aa87613c 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -206,7 +206,7 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi var self = this; if (this.affectedTablesAndLastUpdate !== null) { - return this.affectedTablesAndLastUpdate; + return callback(null, this.affectedTablesAndLastUpdate); } step( From 1f7daab677cec65a878c36f47da784f86c24b86d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 21:16:49 +0200 Subject: [PATCH 26/57] Caching named map providers by template name and config/auth token Named Map provider cache buster changes on creation --- lib/cartodb/cache/named_map_provider_cache.js | 47 ++++++++++++++----- .../models/mapconfig/named_map_provider.js | 6 ++- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index 3acde0e3..c77d3f7b 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -1,24 +1,49 @@ var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); +var templateName = require('../backends/template_maps').templateName; function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.userLimitsApi = userLimitsApi; this.queryTablesApi = queryTablesApi; + + this.providerCache = {}; } module.exports = NamedMapProviderCache; NamedMapProviderCache.prototype.get = function(user, templateId, config, authToken, params) { - return new NamedMapMapConfigProvider( - this.templateMaps, - this.pgConnection, - this.userLimitsApi, - this.queryTablesApi, - user, - templateId, - config, - authToken, - params - ); + var namedMapKey = createNamedMapKey(user, templateId); + if (!this.providerCache.hasOwnProperty(namedMapKey)) { + this.providerCache[namedMapKey] = {}; + } + + var providerKey = createProviderKey(config, authToken); + if (!this.providerCache[namedMapKey].hasOwnProperty(providerKey)) { + this.providerCache[namedMapKey][providerKey] = new NamedMapMapConfigProvider( + this.templateMaps, + this.pgConnection, + this.userLimitsApi, + this.queryTablesApi, + user, + templateId, + config, + authToken, + params + ); + } + + return this.providerCache[namedMapKey][providerKey]; }; + +NamedMapProviderCache.prototype.invalidate = function(user, templateId) { + delete this.providerCache[createNamedMapKey(user, templateId)]; +}; + +function createNamedMapKey(user, templateId) { + return user + ':' + templateName(templateId); +} + +function createProviderKey(config, authToken) { + return NamedMapMapConfigProvider.configHash(config) + ':' + authToken; +} diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index aa87613c..59aa1708 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -22,6 +22,8 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, qu this.authToken = authToken; this.params = params; + this.cacheBuster = Date.now(); + // use template after call to mapConfig this.template = null; @@ -147,7 +149,7 @@ NamedMapMapConfigProvider.prototype.getKey = function() { }; NamedMapMapConfigProvider.prototype.getCacheBuster = function() { - return 0; + return this.cacheBuster; }; NamedMapMapConfigProvider.prototype.filter = function(key) { @@ -182,6 +184,8 @@ function configHash(config) { return crypto.createHash('md5').update(JSON.stringify(config)).digest('hex').substring(0,8); } +module.exports.configHash = configHash; + NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, callback) { var self = this; step( From bbec3ae7dacebaa86027cf374b2a56da23bcb4f0 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 14 Jul 2015 21:18:10 +0200 Subject: [PATCH 27/57] Subscribes to named map changes to invalidate cache --- lib/cartodb/server.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index ebb1affd..e7c74cd0 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -167,6 +167,9 @@ module.exports = function(serverOptions) { app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi); + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); + }); var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); From 7a5928d9572408cd2424287406b6fd1c0872f53d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Jul 2015 11:31:26 +0200 Subject: [PATCH 28/57] Upgrades windshaft to 0.48.0 --- NEWS.md | 5 ++++- npm-shrinkwrap.json | 38 +++++++++++++++++++------------------- package.json | 4 ++-- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/NEWS.md b/NEWS.md index fd371c71..2f4c5171 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,12 @@ # Changelog -## 2.7.3 +## 2.8.0 Released 2015-mm-dd +Announcements: + - Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0) + ## 2.7.2 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 629e1dd2..0e18c3c5 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.7.3", + "version": "2.8.0", "dependencies": { "cartodb-psql": { "version": "0.4.0", @@ -139,14 +139,14 @@ "resolved": "https://registry.npmjs.org/async/-/async-1.3.0.tgz" }, "mime-types": { - "version": "2.1.2", + "version": "2.1.3", "from": "mime-types@^2.1.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.2.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.3.tgz", "dependencies": { "mime-db": { - "version": "1.14.0", - "from": "mime-db@~1.14.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.14.0.tgz" + "version": "1.15.0", + "from": "mime-db@~1.15.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.15.0.tgz" } } } @@ -276,9 +276,9 @@ "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz", "dependencies": { "bluebird": { - "version": "2.9.32", + "version": "2.9.33", "from": "bluebird@^2.9.30", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.32.tgz" + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.33.tgz" }, "chalk": { "version": "1.1.0", @@ -485,9 +485,9 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "0.47.0", - "from": "windshaft@0.47.0", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.47.0.tgz", + "version": "0.48.0", + "from": "windshaft@0.48.0", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.48.0.tgz", "dependencies": { "chronograph": { "version": "0.1.0", @@ -2293,8 +2293,8 @@ }, "tilelive-mapnik": { "version": "0.6.15", - "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/cdb", - "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/cdb", + "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik", + "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik", "dependencies": { "generic-pool": { "version": "2.1.1", @@ -2303,15 +2303,15 @@ }, "mime": { "version": "1.2.11", - "from": "mime@~1.2.9", + "from": "mime@~1.2.11", "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" } } }, "mapnik": { - "version": "1.4.15-cdb1", - "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb1", - "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb1", + "version": "1.4.15-cdb2", + "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb2", + "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb2", "dependencies": { "nan": { "version": "1.2.0", @@ -2919,8 +2919,8 @@ }, "abaculus": { "version": "1.1.0", - "from": "https://github.com/CartoDB/abaculus/tarball/cdb", - "resolved": "https://github.com/CartoDB/abaculus/tarball/cdb" + "from": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik", + "resolved": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik" }, "sphericalmercator": { "version": "1.0.2", diff --git a/package.json b/package.json index e4cb2b75..cfa4bec9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.7.3", + "version": "2.8.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -24,7 +24,7 @@ "dependencies": { "underscore" : "~1.6.0", "dot": "~1.0.2", - "windshaft": "0.47.0", + "windshaft": "0.48.0", "step": "~0.0.5", "queue-async": "~1.0.7", "request": "~2.9.203", From f6c47bf85e2ddeac3437f79a1fead140e6b361b9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Jul 2015 11:43:54 +0200 Subject: [PATCH 29/57] Release 2.8.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2f4c5171..80f4f0a1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## 2.8.0 -Released 2015-mm-dd +Released 2015-07-15 Announcements: - Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0) From decd9077e4b9b1105bc43784ad0faafbac8e6d14 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Jul 2015 11:44:59 +0200 Subject: [PATCH 30/57] Stubs next version --- NEWS.md | 5 +++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 80f4f0a1..5b6aa143 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 2.8.1 + +Released 2015-mm-dd + + ## 2.8.0 Released 2015-07-15 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 0e18c3c5..f6a2263b 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.8.0", + "version": "2.8.1", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index cfa4bec9..2c3423c8 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.8.0", + "version": "2.8.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 9c6c63c167fb14a25f5fb790faf1b0a86654d62a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Jul 2015 15:03:28 +0200 Subject: [PATCH 31/57] More strict jshint --- .jshintrc | 4 +- lib/cartodb/api/auth_api.js | 2 +- lib/cartodb/backends/pg_connection.js | 10 +- lib/cartodb/backends/template_maps.js | 26 +- lib/cartodb/controllers/layergroup.js | 2 - lib/cartodb/controllers/named_maps_admin.js | 4 +- .../models/mapconfig/named_map_provider.js | 2 +- test/acceptance/multilayer.js | 261 +++++++---- test/acceptance/server.js | 2 +- test/acceptance/templates.js | 434 ++++++++++++------ test/unit/cartodb/template_maps.test.js | 76 +-- 11 files changed, 540 insertions(+), 283 deletions(-) diff --git a/.jshintrc b/.jshintrc index f03eef65..8972be0b 100644 --- a/.jshintrc +++ b/.jshintrc @@ -7,8 +7,8 @@ // // Enforcing // "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.) // "camelcase" : false, // true: Identifiers must be in camelCase -// "curly" : true, // true: Require {} for every new block or scope -// "eqeqeq" : true, // true: Require triple equals (===) for comparison + "curly" : true, // true: Require {} for every new block or scope + "eqeqeq" : true, // true: Require triple equals (===) for comparison "forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty() "freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc. "immed" : true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());` diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js index 5b20cbf4..68533fe9 100644 --- a/lib/cartodb/api/auth_api.js +++ b/lib/cartodb/api/auth_api.js @@ -72,7 +72,7 @@ AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) { }, function checkApiKey(err, val){ assert.ifError(err); - return val && givenKey == val; + return val && givenKey === val; }, function finish(err, authorized) { callback(err, authorized); diff --git a/lib/cartodb/backends/pg_connection.js b/lib/cartodb/backends/pg_connection.js index 6cc432fb..2ca56f5b 100644 --- a/lib/cartodb/backends/pg_connection.js +++ b/lib/cartodb/backends/pg_connection.js @@ -37,7 +37,9 @@ PgConnection.prototype.setDBAuth = function(username, params, callback) { // skip looking up user_password if postgres_auth_pass // doesn't contain the "user_password" label - if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null; + if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) { + return null; + } self.metadataBackend.getUserDBPass(username, this); }, @@ -84,10 +86,12 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) { function extendParams(err, dbParams){ assert.ifError(err); // we don't want null values or overwrite a non public user - if (params.dbuser != 'publicuser' || !dbParams.dbuser) { + if (params.dbuser !== 'publicuser' || !dbParams.dbuser) { delete dbParams.dbuser; } - if ( dbParams ) _.extend(params, dbParams); + if ( dbParams ) { + _.extend(params, dbParams); + } return null; }, function finish(err) { diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index 544bdf1f..b550cb53 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -22,7 +22,9 @@ var util = require('util'); // // function TemplateMaps(redis_pool, opts) { - if (!(this instanceof TemplateMaps)) return new TemplateMaps(); + if (!(this instanceof TemplateMaps)) { + return new TemplateMaps(); + } EventEmitter.call(this); @@ -83,7 +85,9 @@ o._redisCmd = function(redisFunc, redisArgs, callback) { redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs); }, function releaseRedisClient(err, data) { - if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient); + if ( ! _.isUndefined(redisClient) ) { + that.redis_pool.release(db, redisClient); + } callback(err, data); } ); @@ -93,7 +97,7 @@ var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i; var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i; // jshint maxcomplexity:15 o._checkInvalidTemplate = function(template) { - if ( template.version != '0.0.1' ) { + if ( template.version !== '0.0.1' ) { return new Error("Unsupported template version " + template.version); } var tplname = template.name; @@ -132,10 +136,12 @@ o._checkInvalidTemplate = function(template) { case 'open': break; case 'token': - if ( ! _.isArray(auth.valid_tokens) ) + if ( ! _.isArray(auth.valid_tokens) ) { return new Error("Invalid 'token' authentication: missing valid_tokens"); - if ( ! auth.valid_tokens.length ) + } + if ( ! auth.valid_tokens.length ) { return new Error("Invalid 'token' authentication: no valid_tokens"); + } break; default: return new Error("Unsupported authentication method: " + auth.method); @@ -301,7 +307,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) { var templateName = template.name; - if ( tpl_id != templateName ) { + if ( tpl_id !== templateName ) { return callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + templateName + "')")); } @@ -463,8 +469,12 @@ o.instance = function(template, params) { var layergroup = JSON.parse(JSON.stringify(template.layergroup)); for (var i=0; i Date: Wed, 15 Jul 2015 15:10:59 +0200 Subject: [PATCH 32/57] More strict cyclomatic complexity check --- .jshintrc | 2 +- test/unit/cartodb/template_maps.test.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.jshintrc b/.jshintrc index 8972be0b..fa18ac74 100644 --- a/.jshintrc +++ b/.jshintrc @@ -31,7 +31,7 @@ // "maxparams" : false, // {int} Max number of formal params allowed per function // "maxdepth" : false, // {int} Max depth of nested blocks (within functions) // "maxstatements" : false, // {int} Max number statements per function - "maxcomplexity" : 8, // {int} Max cyclomatic complexity per function + "maxcomplexity" : 6, // {int} Max cyclomatic complexity per function "maxlen" : 120, // {int} Max number of characters per line // // // Relaxing diff --git a/test/unit/cartodb/template_maps.test.js b/test/unit/cartodb/template_maps.test.js index 1f9792d1..472d57b8 100644 --- a/test/unit/cartodb/template_maps.test.js +++ b/test/unit/cartodb/template_maps.test.js @@ -360,6 +360,7 @@ describe('template_maps', function() { }); it('instanciate templates', function() { + // jshint maxcomplexity:7 var tmap = new TemplateMaps(redis_pool); assert.ok(tmap); From 52b60a22fd6c48549f20fdbea34b55135e704a14 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Jul 2015 16:09:43 +0200 Subject: [PATCH 33/57] Makes all tests to run together --- Makefile | 12 ++++-------- .../ported/support/ported_server_options.js | 4 ++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b572bb2b..3f6ebf43 100644 --- a/Makefile +++ b/Makefile @@ -25,13 +25,9 @@ test: config/environments/test.js test/unit/cartodb/cache/model/*.js \ test/integration/*.js \ test/acceptance/*.js \ - test/acceptance/cache/*.js - -test-ported: config/environments/test.js - @echo "***tests ported***" - @$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \ - test/unit/cartodb/ported/*.js \ - test/acceptance/ported/*.js + test/acceptance/cache/*.js \ + test/acceptance/ported/*.js \ + test/unit/cartodb/ported/*.js test-unit: config/environments/test.js @echo "***tests***" @@ -54,7 +50,7 @@ jshint: @echo "***jshint***" @./node_modules/.bin/jshint lib/ test/ app.js -test-all: jshint test test-ported +test-all: jshint test coverage: @RUNTESTFLAGS=--with-coverage make test diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index b17eef6d..b4ca6a5a 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -58,6 +58,10 @@ module.exports = _.extend({}, serverOptions, { _.extend(req.params, req.query); req.params.user = 'localhost'; req.context = {user: 'localhost'}; + + req.params.dbhost = global.environment.postgres.host; + req.params.dbport = req.params.dbport || global.environment.postgres.port; + req.params.dbuser = 'test_windshaft_publicuser'; if (req.params.dbname !== 'windshaft_test2') { req.params.dbuser = 'test_windshaft_cartodb_user_1'; From 5a5832394a43daaf448fb2981a2412701e7ff25a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Jul 2015 16:10:59 +0200 Subject: [PATCH 34/57] Remove console.log --- lib/cartodb/controllers/layergroup.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index dfcc5ccf..e0aa6b44 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -95,8 +95,6 @@ LayergroupController.prototype.layer = function(req, res, next) { LayergroupController.prototype.tileOrLayer = function (req, res) { var self = this; - console.log(req.context.user); - step( function mapController$prepareParams() { self.app.req2params(req, this); From 909f8da2ff4ccbd42bffd7f72e11a95179d9f94e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Jul 2015 16:51:26 +0200 Subject: [PATCH 35/57] Adds lru cache for layergroups and named maps mapconfig provider --- lib/cartodb/cache/layergroup_affected_tables.js | 12 +++++++----- lib/cartodb/cache/named_map_provider_cache.js | 17 +++++++++-------- npm-shrinkwrap.json | 5 +++++ package.json | 1 + test/acceptance/multilayer_server.js | 2 +- test/acceptance/templates.js | 4 ++-- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/cache/layergroup_affected_tables.js b/lib/cartodb/cache/layergroup_affected_tables.js index c2cccc71..d33c5a73 100644 --- a/lib/cartodb/cache/layergroup_affected_tables.js +++ b/lib/cartodb/cache/layergroup_affected_tables.js @@ -1,20 +1,22 @@ +var LruCache = require('lru-cache'); + function LayergroupAffectedTables() { - // layergroupId -> affected tables cache - this.cache = {}; + // dbname + layergroupId -> affected tables cache + this.cache = new LruCache({ max: 2000 }); } module.exports = LayergroupAffectedTables; LayergroupAffectedTables.prototype.hasAffectedTables = function(dbName, layergroupId) { - return this.cache.hasOwnProperty(createKey(dbName, layergroupId)); + return this.cache.has(createKey(dbName, layergroupId)); }; LayergroupAffectedTables.prototype.set = function(dbName, layergroupId, affectedTables) { - this.cache[createKey(dbName, layergroupId)] = affectedTables; + this.cache.set(createKey(dbName, layergroupId), affectedTables); }; LayergroupAffectedTables.prototype.get = function(dbName, layergroupId) { - return this.cache[createKey(dbName, layergroupId)]; + return this.cache.get(createKey(dbName, layergroupId)); }; function createKey(dbName, layergroupId) { diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index c77d3f7b..49dc6e49 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -1,26 +1,26 @@ var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var templateName = require('../backends/template_maps').templateName; +var LruCache = require("lru-cache"); + function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.userLimitsApi = userLimitsApi; this.queryTablesApi = queryTablesApi; - this.providerCache = {}; + this.providerCache = new LruCache({ max: 2000 }); } module.exports = NamedMapProviderCache; NamedMapProviderCache.prototype.get = function(user, templateId, config, authToken, params) { var namedMapKey = createNamedMapKey(user, templateId); - if (!this.providerCache.hasOwnProperty(namedMapKey)) { - this.providerCache[namedMapKey] = {}; - } + var namedMapProviders = this.providerCache.get(namedMapKey) || {}; var providerKey = createProviderKey(config, authToken); - if (!this.providerCache[namedMapKey].hasOwnProperty(providerKey)) { - this.providerCache[namedMapKey][providerKey] = new NamedMapMapConfigProvider( + if (!namedMapProviders.hasOwnProperty(providerKey)) { + namedMapProviders[providerKey] = new NamedMapMapConfigProvider( this.templateMaps, this.pgConnection, this.userLimitsApi, @@ -31,13 +31,14 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok authToken, params ); + this.providerCache.set(namedMapKey, namedMapProviders); } - return this.providerCache[namedMapKey][providerKey]; + return namedMapProviders[providerKey]; }; NamedMapProviderCache.prototype.invalidate = function(user, templateId) { - delete this.providerCache[createNamedMapKey(user, templateId)]; + this.providerCache.del(createNamedMapKey(user, templateId)); }; function createNamedMapKey(user, templateId) { diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 19a0d623..dba561e5 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -460,6 +460,11 @@ } } }, + "lru-cache": { + "version": "2.6.5", + "from": "lru-cache@", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.6.5.tgz" + }, "lzma": { "version": "1.3.7", "from": "lzma@~1.3.7", diff --git a/package.json b/package.json index d10c62dd..847b8341 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "cartodb-psql": "~0.4.0", "fastly-purge": "~1.0.0", "redis-mpool": "~0.4.0", + "lru-cache": "2.6.5", "lzma": "~1.3.7", "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb" }, diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index 463d41d6..b8b5db2a 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -346,7 +346,7 @@ describe('tests from old api translated to multilayer', function() { }; // reset internal cacheChannel cache - server.layergroupAffectedTablesCache.cache = {}; + server.layergroupAffectedTablesCache.cache.reset(); assert.response(server, { diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 925a6974..c1fa248b 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -21,7 +21,7 @@ var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); describe('template_api', function() { - server.layergroupAffectedTablesCache.cache = {}; + server.layergroupAffectedTablesCache.cache.reset(); var redis_client = redis.createClient(global.environment.redis.port); @@ -1225,7 +1225,7 @@ describe('template_api', function() { assert.ok(cc); assert.ok(cc.match, /ciao/, cc); // hack simulating restart... - server.layergroupAffectedTablesCache.cache = {}; // need to clean channel cache + server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache var get_request = { url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1', method: 'GET', From 2ac228359f3a0b5c6e13acc527d403f2539ead9e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 31 Jul 2015 12:23:36 +0200 Subject: [PATCH 36/57] Fallback to image/png header --- lib/cartodb/controllers/named_maps.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 6936332f..8dbc27d1 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -27,7 +27,7 @@ NamedMapsController.prototype.register = function(app) { NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) { this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName())); - res.header('Content-Type', headers['Content-Type']); + res.header('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png'); res.header('Cache-Control', 'public,max-age=7200,must-revalidate'); var self = this; From 943509864d42bcd8d0808db591252a148acbd3b9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 31 Jul 2015 12:24:34 +0200 Subject: [PATCH 37/57] Improve uniqueness of named map map config provider --- lib/cartodb/cache/named_map_provider_cache.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index 49dc6e49..e4c3dccc 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -1,3 +1,5 @@ +var _ = require('underscore'); +var dot = require('dot'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var templateName = require('../backends/template_maps').templateName; @@ -18,7 +20,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok var namedMapKey = createNamedMapKey(user, templateId); var namedMapProviders = this.providerCache.get(namedMapKey) || {}; - var providerKey = createProviderKey(config, authToken); + var providerKey = createProviderKey(config, authToken, params); if (!namedMapProviders.hasOwnProperty(providerKey)) { namedMapProviders[providerKey] = new NamedMapMapConfigProvider( this.templateMaps, @@ -45,6 +47,16 @@ function createNamedMapKey(user, templateId) { return user + ':' + templateName(templateId); } -function createProviderKey(config, authToken) { - return NamedMapMapConfigProvider.configHash(config) + ':' + authToken; +var providerKey = '{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}'; +var providerKeyTpl = dot.template(providerKey); + +function createProviderKey(config, authToken, params) { + var tplValues = _.defaults({}, params, { + authToken: authToken || '', + configHash: NamedMapMapConfigProvider.configHash(config), + layer: '', + format: '', + scale_factor: 1 + }); + return providerKeyTpl(tplValues); } From 2e063cc2d23e389d996ead10b758411cdd80e550 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 6 Aug 2015 16:06:42 +0200 Subject: [PATCH 38/57] Send memory usage stats --- NEWS.md | 5 ++++- app.js | 7 +++++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 5b6aa143..c3631153 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,12 @@ # Changelog -## 2.8.1 +## 2.9.0 Released 2015-mm-dd +New features: + - Send memory usage stats + ## 2.8.0 diff --git a/app.js b/app.js index 152a6374..0c8ce467 100755 --- a/app.js +++ b/app.js @@ -86,6 +86,13 @@ if (global.statsClient) { global.statsClient.gauge(keyPrefix + 'unused', status.unused); global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); }); + + setInterval(function() { + var memoryUsage = process.memoryUsage(); + Object.keys(memoryUsage).forEach(function(k) { + global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]); + }); + }, 5000); } // Maximum number of connections for one process diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index f6a2263b..a5b647ee 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.8.1", + "version": "2.9.0", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index 2c3423c8..3ecf61a8 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.8.1", + "version": "2.9.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From e3d5abc9a23a8a89e4c528367a28e18e8dfd0732 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 6 Aug 2015 18:02:09 +0200 Subject: [PATCH 39/57] Release 2.9.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index c3631153..60ab786e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## 2.9.0 -Released 2015-mm-dd +Released 2015-08-06 New features: - Send memory usage stats From de2719b0c5eba386efa10e903bc24d911f35bd26 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 6 Aug 2015 18:03:00 +0200 Subject: [PATCH 40/57] Stubs next version --- NEWS.md | 5 +++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 60ab786e..a9f97f23 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 2.9.1 + +Released 2015-mm-dd + + ## 2.9.0 Released 2015-08-06 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index a5b647ee..a3f1dab4 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.9.0", + "version": "2.9.1", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index 3ecf61a8..f15e0215 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.9.0", + "version": "2.9.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 37dfd8fc123f30164c6c2acd90473b7dfafe59d8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 18 Aug 2015 15:18:44 +0200 Subject: [PATCH 41/57] Upgrade windshaft --- npm-shrinkwrap.json | 621 ++++++++++++++++++++++---------------------- package.json | 2 +- 2 files changed, 318 insertions(+), 305 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index a3f1dab4..6a4bc6fe 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -76,52 +76,62 @@ "from": "fastly-purge@~1.0.0", "dependencies": { "request": { - "version": "2.58.0", + "version": "2.60.0", "from": "request@^2.55.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.58.0.tgz", + "resolved": "https://registry.npmjs.org/request/-/request-2.60.0.tgz", "dependencies": { "bl": { - "version": "0.9.4", - "from": "bl@~0.9.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-0.9.4.tgz", + "version": "1.0.0", + "from": "bl@~1.0.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.0.tgz", "dependencies": { "readable-stream": { - "version": "1.0.33", - "from": "readable-stream@~1.0.26", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.33.tgz", + "version": "2.0.2", + "from": "readable-stream@~2.0.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.2.tgz", "dependencies": { "core-util-is": { "version": "1.0.1", "from": "core-util-is@~1.0.0", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" }, + "inherits": { + "version": "2.0.1", + "from": "inherits@~2.0.1" + }, "isarray": { "version": "0.0.1", "from": "isarray@0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" }, + "process-nextick-args": { + "version": "1.0.2", + "from": "process-nextick-args@~1.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.2.tgz" + }, "string_decoder": { "version": "0.10.31", "from": "string_decoder@~0.10.x", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, - "inherits": { - "version": "2.0.1", - "from": "inherits@~2.0.1" + "util-deprecate": { + "version": "1.0.1", + "from": "util-deprecate@~1.0.1", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.1.tgz" } } } } }, "caseless": { - "version": "0.10.0", - "from": "caseless@~0.10.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.10.0.tgz" + "version": "0.11.0", + "from": "caseless@~0.11.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz" }, "extend": { - "version": "2.0.1", - "from": "extend@~2.0.1", - "resolved": "https://registry.npmjs.org/extend/-/extend-2.0.1.tgz" + "version": "3.0.0", + "from": "extend@~3.0.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" }, "forever-agent": { "version": "0.6.1", @@ -129,26 +139,14 @@ "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" }, "form-data": { - "version": "1.0.0-rc1", + "version": "1.0.0-rc3", "from": "form-data@~1.0.0-rc1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc1.tgz", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc3.tgz", "dependencies": { "async": { - "version": "1.3.0", - "from": "async@^1.2.1", - "resolved": "https://registry.npmjs.org/async/-/async-1.3.0.tgz" - }, - "mime-types": { - "version": "2.1.3", - "from": "mime-types@^2.1.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.3.tgz", - "dependencies": { - "mime-db": { - "version": "1.15.0", - "from": "mime-db@~1.15.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.15.0.tgz" - } - } + "version": "1.4.2", + "from": "async@^1.4.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.4.2.tgz" } } }, @@ -158,14 +156,14 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" }, "mime-types": { - "version": "2.0.14", - "from": "mime-types@~2.0.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.14.tgz", + "version": "2.1.4", + "from": "mime-types@~2.1.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.4.tgz", "dependencies": { "mime-db": { - "version": "1.12.0", - "from": "mime-db@~1.12.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.12.0.tgz" + "version": "1.16.0", + "from": "mime-db@~1.16.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.16.0.tgz" } } }, @@ -175,9 +173,9 @@ "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.3.tgz" }, "qs": { - "version": "3.1.0", - "from": "qs@~3.1.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-3.1.0.tgz" + "version": "4.0.0", + "from": "qs@~4.0.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz" }, "tunnel-agent": { "version": "0.4.1", @@ -217,9 +215,9 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.0.tgz" }, "hawk": { - "version": "2.3.1", - "from": "hawk@~2.3.0", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-2.3.1.tgz", + "version": "3.1.0", + "from": "hawk@~3.1.0", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.0.tgz", "dependencies": { "hoek": { "version": "2.14.0", @@ -228,7 +226,7 @@ }, "boom": { "version": "2.8.0", - "from": "boom@2.x.x", + "from": "boom@^2.8.x", "resolved": "https://registry.npmjs.org/boom/-/boom-2.8.0.tgz" }, "cryptiles": { @@ -276,9 +274,9 @@ "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz", "dependencies": { "bluebird": { - "version": "2.9.33", + "version": "2.9.34", "from": "bluebird@^2.9.30", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.33.tgz" + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.34.tgz" }, "chalk": { "version": "1.1.0", @@ -339,9 +337,9 @@ } }, "is-my-json-valid": { - "version": "2.12.0", + "version": "2.12.1", "from": "is-my-json-valid@^2.12.0", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.0.tgz", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.1.tgz", "dependencies": { "generate-function": { "version": "2.0.0", @@ -410,7 +408,7 @@ }, "inherits": { "version": "2.0.1", - "from": "inherits@~2.0.1" + "from": "inherits@2" } } }, @@ -485,9 +483,9 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "0.48.0", - "from": "windshaft@0.48.0", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.48.0.tgz", + "version": "0.49.0", + "from": "windshaft@0.49.0", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.49.0.tgz", "dependencies": { "chronograph": { "version": "0.1.0", @@ -675,9 +673,9 @@ } }, "srs": { - "version": "0.4.8", + "version": "0.4.9", "from": "srs@~0.4.1", - "resolved": "https://registry.npmjs.org/srs/-/srs-0.4.8.tgz", + "resolved": "https://registry.npmjs.org/srs/-/srs-0.4.9.tgz", "dependencies": { "nan": { "version": "1.8.4", @@ -685,49 +683,49 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-1.8.4.tgz" }, "node-pre-gyp": { - "version": "0.6.7", - "from": "node-pre-gyp@~0.6.7", + "version": "0.6.9", + "from": "node-pre-gyp@>=0.6.7 <0.7.0", "dependencies": { "nopt": { - "version": "3.0.1", - "from": "nopt@>=3.0.1 <3.1.0", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.1.tgz", + "version": "3.0.3", + "from": "nopt@~3.0.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.3.tgz", "dependencies": { "abbrev": { - "version": "1.0.5", - "from": "abbrev@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.5.tgz" + "version": "1.0.7", + "from": "abbrev@1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.7.tgz" } } }, "npmlog": { - "version": "1.2.0", - "from": "npmlog@>=1.2.0 <1.3.0", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-1.2.0.tgz", + "version": "1.2.1", + "from": "npmlog@~1.2.0", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-1.2.1.tgz", "dependencies": { "ansi": { "version": "0.3.0", - "from": "ansi@>=0.3.0 <0.4.0", + "from": "ansi@~0.3.0", "resolved": "https://registry.npmjs.org/ansi/-/ansi-0.3.0.tgz" }, "are-we-there-yet": { "version": "1.0.4", - "from": "are-we-there-yet@>=1.0.0 <1.1.0", + "from": "are-we-there-yet@~1.0.0", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.0.4.tgz", "dependencies": { "delegates": { "version": "0.1.0", - "from": "delegates@>=0.1.0 <0.2.0", + "from": "delegates@^0.1.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-0.1.0.tgz" }, "readable-stream": { "version": "1.1.13", - "from": "readable-stream@>=1.1.13 <2.0.0", + "from": "readable-stream@^1.1.13", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.13.tgz", "dependencies": { "core-util-is": { "version": "1.0.1", - "from": "core-util-is@>=1.0.0 <1.1.0", + "from": "core-util-is@~1.0.0", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" }, "isarray": { @@ -737,12 +735,12 @@ }, "string_decoder": { "version": "0.10.31", - "from": "string_decoder@>=0.10.0 <0.11.0", + "from": "string_decoder@~0.10.x", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.1 <2.1.0", + "from": "inherits@~2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" } } @@ -750,34 +748,34 @@ } }, "gauge": { - "version": "1.2.0", - "from": "gauge@>=1.2.0 <1.3.0", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-1.2.0.tgz", + "version": "1.2.2", + "from": "gauge@~1.2.0", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-1.2.2.tgz", "dependencies": { "has-unicode": { "version": "1.0.0", - "from": "has-unicode@>=1.0.0 <2.0.0", + "from": "has-unicode@^1.0.0", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-1.0.0.tgz" }, "lodash.pad": { - "version": "3.1.0", - "from": "lodash.pad@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-3.1.0.tgz", + "version": "3.1.1", + "from": "lodash.pad@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-3.1.1.tgz", "dependencies": { "lodash._basetostring": { - "version": "3.0.0", - "from": "lodash._basetostring@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.0.tgz" + "version": "3.0.1", + "from": "lodash._basetostring@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.1.tgz" }, "lodash._createpadding": { - "version": "3.6.0", - "from": "lodash._createpadding@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.0.tgz", + "version": "3.6.1", + "from": "lodash._createpadding@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.1.tgz", "dependencies": { "lodash.repeat": { - "version": "3.0.0", - "from": "lodash.repeat@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.0.tgz" + "version": "3.0.1", + "from": "lodash.repeat@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.1.tgz" } } } @@ -785,23 +783,23 @@ }, "lodash.padleft": { "version": "3.1.1", - "from": "lodash.padleft@>=3.0.0 <4.0.0", + "from": "lodash.padleft@^3.0.0", "resolved": "https://registry.npmjs.org/lodash.padleft/-/lodash.padleft-3.1.1.tgz", "dependencies": { "lodash._basetostring": { - "version": "3.0.0", - "from": "lodash._basetostring@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.0.tgz" + "version": "3.0.1", + "from": "lodash._basetostring@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.1.tgz" }, "lodash._createpadding": { - "version": "3.6.0", - "from": "lodash._createpadding@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.0.tgz", + "version": "3.6.1", + "from": "lodash._createpadding@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.1.tgz", "dependencies": { "lodash.repeat": { - "version": "3.0.0", - "from": "lodash.repeat@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.0.tgz" + "version": "3.0.1", + "from": "lodash.repeat@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.1.tgz" } } } @@ -809,23 +807,23 @@ }, "lodash.padright": { "version": "3.1.1", - "from": "lodash.padright@>=3.0.0 <4.0.0", + "from": "lodash.padright@^3.0.0", "resolved": "https://registry.npmjs.org/lodash.padright/-/lodash.padright-3.1.1.tgz", "dependencies": { "lodash._basetostring": { - "version": "3.0.0", - "from": "lodash._basetostring@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.0.tgz" + "version": "3.0.1", + "from": "lodash._basetostring@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-3.0.1.tgz" }, "lodash._createpadding": { - "version": "3.6.0", - "from": "lodash._createpadding@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.0.tgz", + "version": "3.6.1", + "from": "lodash._createpadding@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash._createpadding/-/lodash._createpadding-3.6.1.tgz", "dependencies": { "lodash.repeat": { - "version": "3.0.0", - "from": "lodash.repeat@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.0.tgz" + "version": "3.0.1", + "from": "lodash.repeat@^3.0.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-3.0.1.tgz" } } } @@ -836,111 +834,126 @@ } }, "request": { - "version": "2.55.0", - "from": "request@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.55.0.tgz", + "version": "2.60.0", + "from": "request@2.x", + "resolved": "https://registry.npmjs.org/request/-/request-2.60.0.tgz", "dependencies": { "bl": { - "version": "0.9.4", - "from": "bl@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-0.9.4.tgz", + "version": "1.0.0", + "from": "bl@~1.0.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.0.tgz", "dependencies": { "readable-stream": { - "version": "1.0.33", - "from": "readable-stream@>=1.0.26 <1.1.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.33.tgz", + "version": "2.0.2", + "from": "readable-stream@~2.0.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.2.tgz", "dependencies": { "core-util-is": { "version": "1.0.1", - "from": "core-util-is@>=1.0.0 <1.1.0", + "from": "core-util-is@~1.0.0", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" }, + "inherits": { + "version": "2.0.1", + "from": "inherits@2", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + }, "isarray": { "version": "0.0.1", "from": "isarray@0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" }, + "process-nextick-args": { + "version": "1.0.2", + "from": "process-nextick-args@~1.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.2.tgz" + }, "string_decoder": { "version": "0.10.31", - "from": "string_decoder@>=0.10.0 <0.11.0", + "from": "string_decoder@~0.10.x", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, - "inherits": { - "version": "2.0.1", - "from": "inherits@>=2.0.1 <2.1.0", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + "util-deprecate": { + "version": "1.0.1", + "from": "util-deprecate@~1.0.1", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.1.tgz" } } } } }, "caseless": { - "version": "0.9.0", - "from": "caseless@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.9.0.tgz" + "version": "0.11.0", + "from": "caseless@~0.11.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz" + }, + "extend": { + "version": "3.0.0", + "from": "extend@~3.0.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" }, "forever-agent": { "version": "0.6.1", - "from": "forever-agent@>=0.6.0 <0.7.0", + "from": "forever-agent@~0.6.0", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" }, "form-data": { - "version": "0.2.0", - "from": "form-data@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.2.0.tgz", + "version": "1.0.0-rc2", + "from": "form-data@~1.0.0-rc1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc2.tgz", "dependencies": { "async": { - "version": "0.9.0", - "from": "async@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.0.tgz" + "version": "1.4.0", + "from": "async@^1.2.1", + "resolved": "https://registry.npmjs.org/async/-/async-1.4.0.tgz" } } }, "json-stringify-safe": { - "version": "5.0.0", - "from": "json-stringify-safe@>=5.0.0 <5.1.0", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.0.tgz" + "version": "5.0.1", + "from": "json-stringify-safe@~5.0.0", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" }, "mime-types": { - "version": "2.0.10", - "from": "mime-types@>=2.0.1 <2.1.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.10.tgz", + "version": "2.1.3", + "from": "mime-types@~2.1.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.3.tgz", "dependencies": { "mime-db": { - "version": "1.8.0", - "from": "mime-db@>=1.8.0 <1.9.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.8.0.tgz" + "version": "1.15.0", + "from": "mime-db@~1.15.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.15.0.tgz" } } }, "node-uuid": { "version": "1.4.3", - "from": "node-uuid@>=1.4.0 <1.5.0", + "from": "node-uuid@~1.4.0", "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.3.tgz" }, "qs": { - "version": "2.4.1", - "from": "qs@>=2.4.0 <2.5.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-2.4.1.tgz" + "version": "4.0.0", + "from": "qs@~4.0.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz" }, "tunnel-agent": { - "version": "0.4.0", - "from": "tunnel-agent@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.0.tgz" + "version": "0.4.1", + "from": "tunnel-agent@~0.4.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.1.tgz" }, "tough-cookie": { - "version": "1.1.0", + "version": "2.0.0", "from": "tough-cookie@>=0.12.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.0.0.tgz" }, "http-signature": { - "version": "0.10.1", - "from": "http-signature@>=0.10.0 <0.11.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.10.1.tgz", + "version": "0.11.0", + "from": "http-signature@~0.11.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.11.0.tgz", "dependencies": { "assert-plus": { "version": "0.1.5", - "from": "assert-plus@>=0.1.5 <0.2.0", + "from": "assert-plus@^0.1.5", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz" }, "asn1": { @@ -956,167 +969,162 @@ } }, "oauth-sign": { - "version": "0.6.0", - "from": "oauth-sign@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.6.0.tgz" + "version": "0.8.0", + "from": "oauth-sign@~0.8.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.0.tgz" }, "hawk": { - "version": "2.3.1", - "from": "hawk@>=2.3.0 <2.4.0", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-2.3.1.tgz", + "version": "3.1.0", + "from": "hawk@~3.1.0", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.0.tgz", "dependencies": { "hoek": { - "version": "2.13.0", - "from": "hoek@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.13.0.tgz" + "version": "2.14.0", + "from": "hoek@2.x.x", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.14.0.tgz" }, "boom": { - "version": "2.7.1", - "from": "boom@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/boom/-/boom-2.7.1.tgz" + "version": "2.8.0", + "from": "boom@^2.8.x", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.8.0.tgz" }, "cryptiles": { "version": "2.0.4", - "from": "cryptiles@>=2.0.0 <3.0.0", + "from": "cryptiles@2.x.x", "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.4.tgz" }, "sntp": { "version": "1.0.9", - "from": "sntp@>=1.0.0 <2.0.0", + "from": "sntp@1.x.x", "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz" } } }, "aws-sign2": { "version": "0.5.0", - "from": "aws-sign2@>=0.5.0 <0.6.0", + "from": "aws-sign2@~0.5.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz" }, "stringstream": { "version": "0.0.4", - "from": "stringstream@>=0.0.4 <0.1.0", + "from": "stringstream@~0.0.4", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.4.tgz" }, "combined-stream": { - "version": "0.0.7", - "from": "combined-stream@>=0.0.5 <0.1.0", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", + "version": "1.0.5", + "from": "combined-stream@~1.0.1", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", "dependencies": { "delayed-stream": { - "version": "0.0.5", - "from": "delayed-stream@0.0.5", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz" + "version": "1.0.0", + "from": "delayed-stream@~1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" } } }, "isstream": { "version": "0.1.2", - "from": "isstream@>=0.1.1 <0.2.0", + "from": "isstream@~0.1.1", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" }, "har-validator": { - "version": "1.7.0", - "from": "har-validator@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.7.0.tgz", + "version": "1.8.0", + "from": "har-validator@^1.6.1", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz", "dependencies": { "bluebird": { - "version": "2.9.25", - "from": "bluebird@>=2.9.25 <3.0.0", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.25.tgz" + "version": "2.9.34", + "from": "bluebird@^2.9.30", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.34.tgz" }, "chalk": { - "version": "1.0.0", - "from": "chalk@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.0.0.tgz", + "version": "1.1.0", + "from": "chalk@^1.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.0.tgz", "dependencies": { "ansi-styles": { - "version": "2.0.1", - "from": "ansi-styles@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.0.1.tgz" + "version": "2.1.0", + "from": "ansi-styles@^2.1.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz" }, "escape-string-regexp": { "version": "1.0.3", - "from": "escape-string-regexp@>=1.0.2 <2.0.0", + "from": "escape-string-regexp@^1.0.2", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.3.tgz" }, "has-ansi": { - "version": "1.0.3", - "from": "has-ansi@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-1.0.3.tgz", + "version": "2.0.0", + "from": "has-ansi@^2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "dependencies": { "ansi-regex": { - "version": "1.1.1", - "from": "ansi-regex@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-1.1.1.tgz" - }, - "get-stdin": { - "version": "4.0.1", - "from": "get-stdin@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz" + "version": "2.0.0", + "from": "ansi-regex@^2.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" } } }, "strip-ansi": { - "version": "2.0.1", - "from": "strip-ansi@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-2.0.1.tgz", + "version": "3.0.0", + "from": "strip-ansi@^3.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz", "dependencies": { "ansi-regex": { - "version": "1.1.1", - "from": "ansi-regex@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-1.1.1.tgz" + "version": "2.0.0", + "from": "ansi-regex@^2.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" } } }, "supports-color": { - "version": "1.3.1", - "from": "supports-color@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-1.3.1.tgz" + "version": "2.0.0", + "from": "supports-color@^2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" } } }, "commander": { "version": "2.8.1", - "from": "commander@>=2.8.1 <3.0.0", + "from": "commander@^2.8.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", "dependencies": { "graceful-readlink": { "version": "1.0.1", - "from": "graceful-readlink@>=1.0.0", + "from": "graceful-readlink@>= 1.0.0", "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz" } } }, "is-my-json-valid": { - "version": "2.10.1", - "from": "is-my-json-valid@>=2.10.1 <3.0.0", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.10.1.tgz", + "version": "2.12.1", + "from": "is-my-json-valid@^2.12.0", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.1.tgz", "dependencies": { "generate-function": { "version": "2.0.0", - "from": "generate-function@>=2.0.0 <3.0.0", + "from": "generate-function@^2.0.0", "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz" }, "generate-object-property": { - "version": "1.1.1", - "from": "generate-object-property@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.1.1.tgz", + "version": "1.2.0", + "from": "generate-object-property@^1.1.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", "dependencies": { "is-property": { "version": "1.0.2", - "from": "is-property@>=1.0.0 <2.0.0", + "from": "is-property@^1.0.0", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz" } } }, "jsonpointer": { "version": "1.1.0", - "from": "jsonpointer@>=1.1.0 <2.0.0", + "from": "jsonpointer@^1.1.0", "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-1.1.0.tgz" }, "xtend": { "version": "4.0.0", - "from": "xtend@>=4.0.0 <5.0.0", + "from": "xtend@^4.0.0", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.0.tgz" } } @@ -1126,42 +1134,42 @@ } }, "semver": { - "version": "4.3.3", - "from": "semver@>=4.3.2 <4.4.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.3.tgz" + "version": "5.0.1", + "from": "semver@~5.0.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.0.1.tgz" }, "tar": { - "version": "2.1.0", - "from": "tar@>=2.1.0 <2.2.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-2.1.0.tgz", + "version": "2.1.1", + "from": "tar@~2.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-2.1.1.tgz", "dependencies": { "block-stream": { - "version": "0.0.7", + "version": "0.0.8", "from": "block-stream@*", - "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.7.tgz" + "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.8.tgz" }, "fstream": { - "version": "1.0.4", - "from": "fstream@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.4.tgz", + "version": "1.0.7", + "from": "fstream@^1.0.2", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.7.tgz", "dependencies": { "graceful-fs": { - "version": "3.0.6", - "from": "graceful-fs@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.6.tgz" + "version": "3.0.8", + "from": "graceful-fs@3", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.8.tgz" } } }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.0 <3.0.0", + "from": "inherits@2", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" } } }, "tar-pack": { "version": "2.0.0", - "from": "tar-pack@>=2.0.0 <2.1.0", + "from": "tar-pack@~2.0.0", "resolved": "https://registry.npmjs.org/tar-pack/-/tar-pack-2.0.0.tgz", "dependencies": { "uid-number": { @@ -1171,49 +1179,49 @@ }, "once": { "version": "1.1.1", - "from": "once@>=1.1.1 <1.2.0", + "from": "once@~1.1.1", "resolved": "https://registry.npmjs.org/once/-/once-1.1.1.tgz" }, "debug": { "version": "0.7.4", - "from": "debug@>=0.7.2 <0.8.0", + "from": "debug@~0.7.2", "resolved": "https://registry.npmjs.org/debug/-/debug-0.7.4.tgz" }, "rimraf": { "version": "2.2.8", - "from": "rimraf@>=2.2.0 <2.3.0", + "from": "rimraf@~2.2.0", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz" }, "fstream": { "version": "0.1.31", - "from": "fstream@>=0.1.22 <0.2.0", + "from": "fstream@~0.1.22", "resolved": "https://registry.npmjs.org/fstream/-/fstream-0.1.31.tgz", "dependencies": { "graceful-fs": { - "version": "3.0.6", - "from": "graceful-fs@>=3.0.2 <3.1.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.6.tgz" + "version": "3.0.8", + "from": "graceful-fs@~3.0.2", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.8.tgz" }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.0 <2.1.0", + "from": "inherits@~2.0.0", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" } } }, "tar": { "version": "0.1.20", - "from": "tar@>=0.1.17 <0.2.0", + "from": "tar@~0.1.17", "resolved": "https://registry.npmjs.org/tar/-/tar-0.1.20.tgz", "dependencies": { "block-stream": { - "version": "0.0.7", + "version": "0.0.8", "from": "block-stream@*", - "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.7.tgz" + "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.8.tgz" }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.0 <3.0.0", + "from": "inherits@2", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" } } @@ -1225,36 +1233,36 @@ "dependencies": { "minimatch": { "version": "0.2.14", - "from": "minimatch@>=0.2.0 <0.3.0", + "from": "minimatch@~0.2.0", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz", "dependencies": { "lru-cache": { - "version": "2.6.2", - "from": "lru-cache@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.6.2.tgz" + "version": "2.6.5", + "from": "lru-cache@2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.6.5.tgz" }, "sigmund": { - "version": "1.0.0", - "from": "sigmund@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.0.tgz" + "version": "1.0.1", + "from": "sigmund@~1.0.0", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz" } } }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.0 <3.0.0", + "from": "inherits@~2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" } } }, "readable-stream": { "version": "1.0.33", - "from": "readable-stream@>=1.0.2 <1.1.0", + "from": "readable-stream@~1.0.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.33.tgz", "dependencies": { "core-util-is": { "version": "1.0.1", - "from": "core-util-is@>=1.0.0 <1.1.0", + "from": "core-util-is@~1.0.0", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" }, "isarray": { @@ -1264,27 +1272,27 @@ }, "string_decoder": { "version": "0.10.31", - "from": "string_decoder@>=0.10.0 <0.11.0", + "from": "string_decoder@~0.10.x", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.1 <2.1.0", + "from": "inherits@~2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" } } }, "graceful-fs": { "version": "1.2.3", - "from": "graceful-fs@>=1.2.0 <1.3.0", + "from": "graceful-fs@1.2", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz" } } }, "mkdirp": { - "version": "0.5.0", - "from": "mkdirp@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz", + "version": "0.5.1", + "from": "mkdirp@~0.5.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "dependencies": { "minimist": { "version": "0.0.8", @@ -1294,72 +1302,72 @@ } }, "rc": { - "version": "1.0.1", - "from": "rc@>=1.0.1 <1.1.0", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.0.1.tgz", + "version": "1.1.0", + "from": "rc@~1.1.0", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.1.0.tgz", "dependencies": { "minimist": { - "version": "0.0.10", - "from": "minimist@>=0.0.7 <0.1.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz" + "version": "1.1.2", + "from": "minimist@^1.1.2", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.1.2.tgz" }, "deep-extend": { "version": "0.2.11", - "from": "deep-extend@>=0.2.5 <0.3.0", + "from": "deep-extend@~0.2.5", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.2.11.tgz" }, "strip-json-comments": { "version": "0.1.3", - "from": "strip-json-comments@>=0.1.0 <0.2.0", + "from": "strip-json-comments@0.1.x", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-0.1.3.tgz" }, "ini": { - "version": "1.3.3", - "from": "ini@>=1.3.0 <1.4.0", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.3.tgz" + "version": "1.3.4", + "from": "ini@~1.3.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.4.tgz" } } }, "rimraf": { - "version": "2.3.3", - "from": "rimraf@>=2.3.2 <2.4.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.3.3.tgz", + "version": "2.4.2", + "from": "rimraf@~2.4.0", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.2.tgz", "dependencies": { "glob": { - "version": "4.5.3", - "from": "glob@>=4.4.2 <5.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-4.5.3.tgz", + "version": "5.0.14", + "from": "glob@^5.0.14", + "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.14.tgz", "dependencies": { "inflight": { "version": "1.0.4", - "from": "inflight@>=1.0.4 <2.0.0", + "from": "inflight@^1.0.4", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.4.tgz", "dependencies": { "wrappy": { "version": "1.0.1", - "from": "wrappy@>=1.0.0 <2.0.0", + "from": "wrappy@1", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" } } }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.0 <3.0.0", + "from": "inherits@~2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" }, "minimatch": { - "version": "2.0.7", - "from": "minimatch@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-2.0.7.tgz", + "version": "2.0.10", + "from": "minimatch@2.0.x", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-2.0.10.tgz", "dependencies": { "brace-expansion": { "version": "1.1.0", - "from": "brace-expansion@>=1.0.0 <2.0.0", + "from": "brace-expansion@^1.0.0", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.0.tgz", "dependencies": { "balanced-match": { "version": "0.2.0", - "from": "balanced-match@>=0.2.0 <0.3.0", + "from": "balanced-match@^0.2.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.2.0.tgz" }, "concat-map": { @@ -1372,16 +1380,21 @@ } }, "once": { - "version": "1.3.1", - "from": "once@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.3.1.tgz", + "version": "1.3.2", + "from": "once@^1.3.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.3.2.tgz", "dependencies": { "wrappy": { "version": "1.0.1", - "from": "wrappy@>=1.0.0 <2.0.0", + "from": "wrappy@1", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" } } + }, + "path-is-absolute": { + "version": "1.0.0", + "from": "path-is-absolute@^1.0.0", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.0.tgz" } } } @@ -2293,8 +2306,8 @@ }, "tilelive-mapnik": { "version": "0.6.15", - "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik", - "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik", + "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/0.6.x-cdb1", + "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/0.6.x-cdb1", "dependencies": { "generic-pool": { "version": "2.1.1", @@ -2303,7 +2316,7 @@ }, "mime": { "version": "1.2.11", - "from": "mime@~1.2.11", + "from": "mime@~1.2.9", "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" } } @@ -2919,8 +2932,8 @@ }, "abaculus": { "version": "1.1.0", - "from": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik", - "resolved": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik" + "from": "https://github.com/CartoDB/abaculus/tarball/cdb", + "resolved": "https://github.com/CartoDB/abaculus/tarball/cdb" }, "sphericalmercator": { "version": "1.0.2", diff --git a/package.json b/package.json index f15e0215..217158b4 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "dependencies": { "underscore" : "~1.6.0", "dot": "~1.0.2", - "windshaft": "0.48.0", + "windshaft": "0.49.0", "step": "~0.0.5", "queue-async": "~1.0.7", "request": "~2.9.203", From 3121ed9a9570efbc9eea7d2996f824c05c2a1cb6 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 18 Aug 2015 15:18:58 +0200 Subject: [PATCH 42/57] Config samples for metatileCache with default behaviour --- config/environments/development.js.example | 11 +++++++++++ config/environments/production.js.example | 11 +++++++++++ config/environments/staging.js.example | 11 +++++++++++ config/environments/test.js.example | 11 +++++++++++ 4 files changed, 44 insertions(+) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 37c09037..760a7840 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -101,6 +101,17 @@ var config = { // wasted time. metatile: 2, + // tilelive-mapnik uses an internal cache to store tiles/grids + // generated when using metatile. This options allow to tune + // the behaviour for that internal cache. + metatileCache: { + // Time an object must stay in the cache until is removed + ttl: 0, + // Whether an object must be removed after the first hit + // Usually you want to use `true` here when ttl>0. + deleteOnHit: false + }, + // Override metatile behaviour depending on the format formatMetatile: { png: 2, diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 64e0e26e..2e31ddf6 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -95,6 +95,17 @@ var config = { // wasted time. metatile: 2, + // tilelive-mapnik uses an internal cache to store tiles/grids + // generated when using metatile. This options allow to tune + // the behaviour for that internal cache. + metatileCache: { + // Time an object must stay in the cache until is removed + ttl: 0, + // Whether an object must be removed after the first hit + // Usually you want to use `true` here when ttl>0. + deleteOnHit: false + }, + // Override metatile behaviour depending on the format formatMetatile: { png: 2, diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 05589ce7..941083bd 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -95,6 +95,17 @@ var config = { // wasted time. metatile: 2, + // tilelive-mapnik uses an internal cache to store tiles/grids + // generated when using metatile. This options allow to tune + // the behaviour for that internal cache. + metatileCache: { + // Time an object must stay in the cache until is removed + ttl: 0, + // Whether an object must be removed after the first hit + // Usually you want to use `true` here when ttl>0. + deleteOnHit: false + }, + // Override metatile behaviour depending on the format formatMetatile: { png: 2, diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 1d8ea9da..65cce732 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -95,6 +95,17 @@ var config = { // wasted time. metatile: 2, + // tilelive-mapnik uses an internal cache to store tiles/grids + // generated when using metatile. This options allow to tune + // the behaviour for that internal cache. + metatileCache: { + // Time an object must stay in the cache until is removed + ttl: 0, + // Whether an object must be removed after the first hit + // Usually you want to use `true` here when ttl>0. + deleteOnHit: false + }, + // Override metatile behaviour depending on the format formatMetatile: { png: 2, From b77be76f5162efd087b6322bc923c8782925d08b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 18 Aug 2015 15:22:27 +0200 Subject: [PATCH 43/57] Bump version and add news --- NEWS.md | 8 +++++++- npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index a9f97f23..481e4554 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,15 @@ # Changelog -## 2.9.1 +## 2.10.0 Released 2015-mm-dd +New features: + - Exposes metatile cache configuration for tilelive-mapnik, see configuration sample files for more information. + +Announcements: + - Upgrades windshaft to [0.49.0](https://github.com/CartoDB/Windshaft/releases/tag/0.49.0) + ## 2.9.0 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 6a4bc6fe..df369323 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.9.1", + "version": "2.10.0", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index 217158b4..55e75fc5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.9.1", + "version": "2.10.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 06f454abcf204993ea2a515a5cae15e870392a0a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 18 Aug 2015 16:33:44 +0200 Subject: [PATCH 44/57] Release 2.10.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 481e4554..404ecbb6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## 2.10.0 -Released 2015-mm-dd +Released 2015-08-18 New features: - Exposes metatile cache configuration for tilelive-mapnik, see configuration sample files for more information. From bf4844e664e83a6bfc16245fe9bdf180fcec86a3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 18 Aug 2015 16:35:02 +0200 Subject: [PATCH 45/57] Stubs next version --- NEWS.md | 5 +++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 404ecbb6..b88627a9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 2.10.1 + +Released 2015-mm-dd + + ## 2.10.0 Released 2015-08-18 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index df369323..ccd3ac42 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.10.0", + "version": "2.10.1", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index 55e75fc5..bb439119 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.10.0", + "version": "2.10.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 9ef96080a60a139712a2692a6e36ca8eeb475884 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Aug 2015 12:45:21 +0200 Subject: [PATCH 46/57] Log PID on start --- app.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app.js b/app.js index 0c8ce467..3844a74e 100755 --- a/app.js +++ b/app.js @@ -106,10 +106,10 @@ ws.listen(global.environment.port, global.environment.host); var version = require("./package").version; ws.on('listening', function() { - console.log( - "Windshaft tileserver %s started on %s:%s (%s)", - version, global.environment.host, global.environment.port, ENV - ); + console.log( + "Windshaft tileserver %s started on %s:%s PID=%d (%s)", + version, global.environment.host, global.environment.port, process.pid, ENV + ); }); process.on('SIGHUP', function() { From 627bc672bca197ee4915feb88640cc4f401a60ed Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 25 Aug 2015 19:27:00 +0200 Subject: [PATCH 47/57] Upgrades windshaft to 0.50.0 --- NEWS.md | 5 ++++- npm-shrinkwrap.json | 40 ++++++++++++++++++++-------------------- package.json | 4 ++-- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/NEWS.md b/NEWS.md index b88627a9..66f3d0b8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,12 @@ # Changelog -## 2.10.1 +## 2.11.0 Released 2015-mm-dd +Announcements: + - Upgrades windshaft to [0.50.0](https://github.com/CartoDB/Windshaft/releases/tag/0.50.0) + ## 2.10.0 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index ccd3ac42..8f4d1cd3 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.10.1", + "version": "2.11.0", "dependencies": { "cartodb-psql": { "version": "0.4.0", @@ -76,9 +76,9 @@ "from": "fastly-purge@~1.0.0", "dependencies": { "request": { - "version": "2.60.0", + "version": "2.61.0", "from": "request@^2.55.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.60.0.tgz", + "resolved": "https://registry.npmjs.org/request/-/request-2.61.0.tgz", "dependencies": { "bl": { "version": "1.0.0", @@ -156,14 +156,14 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" }, "mime-types": { - "version": "2.1.4", + "version": "2.1.5", "from": "mime-types@~2.1.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.4.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.5.tgz", "dependencies": { "mime-db": { - "version": "1.16.0", - "from": "mime-db@~1.16.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.16.0.tgz" + "version": "1.17.0", + "from": "mime-db@~1.17.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.17.0.tgz" } } }, @@ -279,9 +279,9 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.34.tgz" }, "chalk": { - "version": "1.1.0", + "version": "1.1.1", "from": "chalk@^1.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz", "dependencies": { "ansi-styles": { "version": "2.1.0", @@ -408,7 +408,7 @@ }, "inherits": { "version": "2.0.1", - "from": "inherits@2" + "from": "inherits@~2.0.1" } } }, @@ -483,9 +483,9 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "0.49.0", - "from": "windshaft@0.49.0", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.49.0.tgz", + "version": "0.50.0", + "from": "windshaft@0.50.0", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.50.0.tgz", "dependencies": { "chronograph": { "version": "0.1.0", @@ -2707,14 +2707,14 @@ } }, "canvas": { - "version": "1.2.1", - "from": "canvas@1.2.1", - "resolved": "https://registry.npmjs.org/canvas/-/canvas-1.2.1.tgz", + "version": "1.2.7-cdb1", + "from": "https://github.com/CartoDB/node-canvas/tarball/1.2.7-cdb1", + "resolved": "https://github.com/CartoDB/node-canvas/tarball/1.2.7-cdb1", "dependencies": { "nan": { - "version": "1.5.3", - "from": "nan@~1.5.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-1.5.3.tgz" + "version": "1.9.0", + "from": "nan@^1.8.4", + "resolved": "https://registry.npmjs.org/nan/-/nan-1.9.0.tgz" } } }, diff --git a/package.json b/package.json index bb439119..3e2df14b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.10.1", + "version": "2.11.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -24,7 +24,7 @@ "dependencies": { "underscore" : "~1.6.0", "dot": "~1.0.2", - "windshaft": "0.49.0", + "windshaft": "0.50.0", "step": "~0.0.5", "queue-async": "~1.0.7", "request": "~2.9.203", From e667d10eb8f5a9110d7e2b2b35eee5389d5f3688 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 26 Aug 2015 09:52:53 +0200 Subject: [PATCH 48/57] Release 2.11.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 66f3d0b8..cd9ec9ba 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## 2.11.0 -Released 2015-mm-dd +Released 2015-08-26 Announcements: - Upgrades windshaft to [0.50.0](https://github.com/CartoDB/Windshaft/releases/tag/0.50.0) From b3467116fe1f156cc01f010f5252b91b72d3e586 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 26 Aug 2015 09:55:15 +0200 Subject: [PATCH 49/57] Stubs next version --- NEWS.md | 5 +++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index cd9ec9ba..2c873cdc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 2.11.1 + +Released 2015-mm-dd + + ## 2.11.0 Released 2015-08-26 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 8f4d1cd3..8382cc3a 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.11.0", + "version": "2.11.1", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index 3e2df14b..07ebdf3a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.11.0", + "version": "2.11.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 0561a28a4d68c5ca2b385d828624ef584c810dd3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 27 Aug 2015 16:28:16 +0200 Subject: [PATCH 50/57] Make http and https globalAgent options configurable --- NEWS.md | 6 +++++- app.js | 13 +++++++++++++ config/environments/development.js.example | 7 +++++++ config/environments/production.js.example | 7 +++++++ config/environments/staging.js.example | 7 +++++++ config/environments/test.js.example | 7 +++++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 8 files changed, 48 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 2c873cdc..175532d7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,13 @@ # Changelog -## 2.11.1 +## 2.12.0 Released 2015-mm-dd +New features: + - Make http and https globalAgent options configurable + * If config is not provided it configures them with default values + ## 2.11.0 diff --git a/app.js b/app.js index 3844a74e..e2651bfd 100755 --- a/app.js +++ b/app.js @@ -9,6 +9,8 @@ var path = require('path'); var fs = require('fs'); +var http = require('http'); +var https = require('https'); var RedisPool = require('redis-mpool'); var _ = require('underscore'); @@ -72,6 +74,17 @@ var redisOpts = _.defaults(global.environment.redis, { }); var redisPool = new RedisPool(redisOpts); +// set global HTTP and HTTPS agent default configurations +// ref https://nodejs.org/api/http.html#http_new_agent_options +var agentOptions = _.defaults(global.environment.httpAgent || {}, { + keepAlive: false, + keepAliveMsecs: 1000, + maxSockets: Infinity, + maxFreeSockets: 256 +}); +http.globalAgent = new http.Agent(agentOptions); +https.globalAgent = new https.Agent(agentOptions); + // Include cartodb_windshaft only _after_ the "global" variable is set // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 var cartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'), diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 760a7840..c3401aa2 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -196,6 +196,13 @@ var config = { unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading } + // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options + ,httpAgent: { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 25, + maxFreeSockets: 256 + } ,varnish: { host: 'localhost', port: 6082, // the por for the telnet interface where varnish is listening to diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 2e31ddf6..d251ac8e 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -190,6 +190,13 @@ var config = { unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading } + // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options + ,httpAgent: { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 25, + maxFreeSockets: 256 + } ,varnish: { host: 'localhost', port: 6082, // the por for the telnet interface where varnish is listening to diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 941083bd..5fe72c9e 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -190,6 +190,13 @@ var config = { unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading } + // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options + ,httpAgent: { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 25, + maxFreeSockets: 256 + } ,varnish: { host: 'localhost', port: 6082, // the por for the telnet interface where varnish is listening to diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 65cce732..f0002aa5 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -192,6 +192,13 @@ var config = { unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading } + // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options + ,httpAgent: { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 25, + maxFreeSockets: 256 + } ,varnish: { host: '', port: null, // the por for the telnet interface where varnish is listening to diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 8382cc3a..99b48dd2 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.11.1", + "version": "2.12.0", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index 07ebdf3a..2854752e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.11.1", + "version": "2.12.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From fa85dcbc4cbc309f8be66ad2f9ca17793997f91a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 27 Aug 2015 17:24:38 +0200 Subject: [PATCH 51/57] Upgrades windshaft to 0.51.0 --- NEWS.md | 3 +++ npm-shrinkwrap.json | 6 +++--- package.json | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 175532d7..8b77c54a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,9 @@ Released 2015-mm-dd +Announcements: + - Upgrades windshaft to [0.51.0](https://github.com/CartoDB/Windshaft/releases/tag/0.51.0) + New features: - Make http and https globalAgent options configurable * If config is not provided it configures them with default values diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 99b48dd2..ba2d5d4d 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -483,9 +483,9 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "0.50.0", - "from": "windshaft@0.50.0", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.50.0.tgz", + "version": "0.51.0", + "from": "windshaft@0.51.0", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.51.0.tgz", "dependencies": { "chronograph": { "version": "0.1.0", diff --git a/package.json b/package.json index 2854752e..43a9daa9 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "dependencies": { "underscore" : "~1.6.0", "dot": "~1.0.2", - "windshaft": "0.50.0", + "windshaft": "0.51.0", "step": "~0.0.5", "queue-async": "~1.0.7", "request": "~2.9.203", From f2f342e14c7ed2d1b95336156d7d7d194a7e8279 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 27 Aug 2015 17:46:29 +0200 Subject: [PATCH 52/57] Release 2.12.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 8b77c54a..9cd5a137 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## 2.12.0 -Released 2015-mm-dd +Released 2015-08-27 Announcements: - Upgrades windshaft to [0.51.0](https://github.com/CartoDB/Windshaft/releases/tag/0.51.0) From bb17609ea3385e54f9ccc58632fbfb161e9a355c Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 27 Aug 2015 17:48:06 +0200 Subject: [PATCH 53/57] Stubs next version --- NEWS.md | 6 ++++++ npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9cd5a137..8d7966aa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 2.12.1 + +Released 2015-mm-dd + + + ## 2.12.0 Released 2015-08-27 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index ba2d5d4d..15763e64 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.12.0", + "version": "2.12.1", "dependencies": { "cartodb-psql": { "version": "0.4.0", diff --git a/package.json b/package.json index 43a9daa9..413d084f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.12.0", + "version": "2.12.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From d6102284a4ed6ae28f2bf671820c8d6e45fa77a6 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 28 Aug 2015 17:41:40 +0200 Subject: [PATCH 54/57] Do not return results from health check It also removed old dependencies and takes disabled file path in ctor. --- lib/cartodb/cartodb_windshaft.js | 7 +++---- lib/cartodb/monitoring/health_check.js | 21 +++++---------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index b9429a2a..31872ede 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -187,19 +187,18 @@ var CartodbWindshaft = function(serverOptions) { * END Routing ******************************************************************************************************************/ - var healthCheck = new HealthCheck(cartoData, Windshaft.tilelive); + var healthCheck = new HealthCheck(global.environment.disabled_file); ws.get('/health', function(req, res) { var healthConfig = global.environment.health || {}; if (!!healthConfig.enabled) { var startTime = Date.now(); - healthCheck.check(healthConfig, function(err, result) { + healthCheck.check(healthConfig, function(err) { var ok = !err; var response = { enabled: true, ok: ok, - elapsed: Date.now() - startTime, - result: result + elapsed: Date.now() - startTime }; if (err) { response.err = err.message; diff --git a/lib/cartodb/monitoring/health_check.js b/lib/cartodb/monitoring/health_check.js index 1fca02b8..dd405149 100644 --- a/lib/cartodb/monitoring/health_check.js +++ b/lib/cartodb/monitoring/health_check.js @@ -1,9 +1,8 @@ var fs = require('fs'); var step = require('step'); -function HealthCheck(metadataBackend, tilelive) { - this.metadataBackend = metadataBackend; - this.tilelive = tilelive; +function HealthCheck(disableFile) { + this.disableFile = disableFile; } module.exports = HealthCheck; @@ -11,21 +10,11 @@ module.exports = HealthCheck; HealthCheck.prototype.check = function(config, callback) { - var result = { - redis: { - ok: false - }, - mapnik: { - ok: false - }, - tile: { - ok: false - } - }; + var self = this; step( function getManualDisable() { - fs.readFile(global.environment.disabled_file, this); + fs.readFile(self.disableFile, this); }, function handleDisabledFile(err, data) { var next = this; @@ -39,7 +28,7 @@ HealthCheck.prototype.check = function(config, callback) { } }, function handleResult(err) { - callback(err, result); + return callback(err); } ); }; From e0a7eb01ccfe7ff4c1d40eba961025c0946161c4 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 4 Sep 2015 16:33:40 +0200 Subject: [PATCH 55/57] Use torque renderer config Adds some notes about db pool params in torque --- config/environments/development.js.example | 10 ++++++++++ config/environments/production.js.example | 10 ++++++++++ config/environments/staging.js.example | 10 ++++++++++ config/environments/test.js.example | 10 ++++++++++ lib/cartodb/server_options.js | 1 + 5 files changed, 41 insertions(+) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index c3401aa2..bf98ca23 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -162,6 +162,16 @@ var config = { type: 'fs', // 'fs' and 'url' supported src: __dirname + '/../../assets/default-placeholder.png' } + }, + torque: { + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } } ,millstone: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index d251ac8e..386f4433 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -156,6 +156,16 @@ var config = { type: 'fs', // 'fs' and 'url' supported src: __dirname + '/../../assets/default-placeholder.png' } + }, + torque: { + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } } ,millstone: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 5fe72c9e..f692d2ba 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -156,6 +156,16 @@ var config = { type: 'fs', // 'fs' and 'url' supported src: __dirname + '/../../assets/default-placeholder.png' } + }, + torque: { + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } } ,millstone: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index f0002aa5..13132e9a 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -158,6 +158,16 @@ var config = { type: 'fs', // 'fs' and 'url' supported src: __dirname + '/../../assets/default-placeholder.png' } + }, + torque: { + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } } ,millstone: { diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 4b2c595d..ac416d30 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -79,6 +79,7 @@ module.exports = function(redisPool) { }, renderer: { mapnik: rendererConfig.mapnik, + torque: rendererConfig.torque, http: rendererConfig.http }, redis: global.environment.redis, From c409c146bf031ab373d8f33a02dafcdb4d5d2e15 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 7 Sep 2015 17:17:40 +0200 Subject: [PATCH 56/57] Upgrade CDB_QueryTables to use latest version --- test/support/sql/CDB_QueryTables.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/sql/CDB_QueryTables.sql b/test/support/sql/CDB_QueryTables.sql index ac61281d..c7cfa64b 100644 --- a/test/support/sql/CDB_QueryTables.sql +++ b/test/support/sql/CDB_QueryTables.sql @@ -41,11 +41,11 @@ BEGIN xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x, xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s ) - SELECT unnest(x) as p, unnest(s) as sc from inp + SELECT unnest(x)::text as p, unnest(s)::text as sc from inp LOOP -- RAISE DEBUG 'tab: %', rec2.p; -- RAISE DEBUG 'sc: %', rec2.sc; - tables := array_append(tables, (rec2.sc || '.' || rec2.p)); + tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p))); END LOOP; -- RAISE DEBUG 'Tables: %', tables; From 7ae034d746388f4393bb2132cad63151e38ad449 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 7 Sep 2015 18:40:20 +0200 Subject: [PATCH 57/57] Remove no longer needed health check params --- lib/cartodb/cartodb_windshaft.js | 2 +- lib/cartodb/monitoring/health_check.js | 2 +- test/acceptance/health_check.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 31872ede..950b0649 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -193,7 +193,7 @@ var CartodbWindshaft = function(serverOptions) { if (!!healthConfig.enabled) { var startTime = Date.now(); - healthCheck.check(healthConfig, function(err) { + healthCheck.check(function(err) { var ok = !err; var response = { enabled: true, diff --git a/lib/cartodb/monitoring/health_check.js b/lib/cartodb/monitoring/health_check.js index dd405149..69425c2e 100644 --- a/lib/cartodb/monitoring/health_check.js +++ b/lib/cartodb/monitoring/health_check.js @@ -8,7 +8,7 @@ function HealthCheck(disableFile) { module.exports = HealthCheck; -HealthCheck.prototype.check = function(config, callback) { +HealthCheck.prototype.check = function(callback) { var self = this; diff --git a/test/acceptance/health_check.js b/test/acceptance/health_check.js index 274e9cfc..4e54393e 100644 --- a/test/acceptance/health_check.js +++ b/test/acceptance/health_check.js @@ -59,7 +59,7 @@ describe('health checks', function () { callback(null, "Maintenance"); }; - healthCheck.check(null, function(err/*, result*/) { + healthCheck.check(function(err) { assert.equal(err.message, "Maintenance"); assert.equal(err.http_status, 503); done();