From 0684c1b9d34ffdee103bfee9f0cdb0697e04f7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 5 Jul 2017 19:08:19 +0200 Subject: [PATCH 001/128] Work in progress: get timeout from redis --- lib/cartodb/api/user_limits_api.js | 88 ++++++++++++++++--- .../provider/create-layergroup-provider.js | 2 +- .../mapconfig/provider/map-store-provider.js | 2 +- .../mapconfig/provider/named-map-provider.js | 2 +- test/acceptance/user-timeout-limit.js | 82 +++++++++++++++++ 5 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 test/acceptance/user-timeout-limit.js diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index ae9cd46b..7f244e7d 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -1,3 +1,5 @@ +var step = require('step'); + /** * * @param metadataBackend @@ -13,16 +15,82 @@ function UserLimitsApi(metadataBackend, options) { module.exports = UserLimitsApi; -UserLimitsApi.prototype.getRenderLimits = function (username, callback) { +UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) { var self = this; - this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) { - if (err) { - return callback(err); - } - return callback(null, { - cacheOnTimeout: self.options.limits.cacheOnTimeout || false, - render: renderLimit || self.options.limits.render || 0 - }); - }); + var limits = { + cacheOnTimeout: self.options.limits.cacheOnTimeout || false, + render: self.options.limits.render || 0 + }; + + step( + function getTilerLimit() { + var next = this; + self.getTilerRenderLimit(username, function (err, tilerRenderLimit) { + if (err) { + return callback(err); + } + + if (Number.isFinite(tilerRenderLimit)) { + limits.render = tilerRenderLimit; + return callback(null, limits); + } + + return next(); + }); + }, + function getTimeoutLimit() { + self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { + if (err) { + return callback(err); + } + + if (timeoutRenderLimit && timeoutRenderLimit.render) { + if (Number.isFinite(timeoutRenderLimit.render)) { + limits.render = timeoutRenderLimit.render; + } + } + + return callback(null, limits); + }); + } + ); +}; + +UserLimitsApi.prototype.getTilerRenderLimit = function (username, callback) { + this.metadataBackend.getTilerRenderLimit(username, callback); +}; + +UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { + var self = this; + + step( + function isAuthorized() { + var next = this; + + if (!apiKey) { + return next(null, false); + } + + self.metadataBackend.getUserMapKey(username, function (err, userApiKey) { + if (err) { + return next(err); + } + + return next(null, userApiKey === apiKey); + }); + }, + function getUserTimeoutRenderLimits(err, authorized) { + if (err) { + return next(err); + } + self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, this); + }, + function setTilerRenderLimit(err, timeoutRenderLimit) { + if (err) { + return callback(err); + } + return callback(null, timeoutRenderLimit); + } + ); }; diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index af5a775b..340073b5 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { var context = {}; step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, this); + self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index c07f9abc..177322d4 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var context = {}; step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, this); + self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 594ec247..17c66c29 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -114,7 +114,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { function prepareContextLimits(err, _mapConfig) { assert.ifError(err); mapConfig = _mapConfig; - self.userLimitsApi.getRenderLimits(self.owner, this); + self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this); }, function cacheAndReturnMapConfig(err, renderLimits) { self.err = err; diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js new file mode 100644 index 00000000..b595cd0e --- /dev/null +++ b/test/acceptance/user-timeout-limit.js @@ -0,0 +1,82 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); +var testHelper = require('../support/test_helper'); + +var redis = require('redis'); +var keysToDelete; + +function withUserTimeoutRenderLimit(redisClient, user, userTimeoutLimit, callback) { + redisClient.SELECT(5, function(err) { + if (err) { + return callback(err); + } + + var userTimeoutLimitsKey = 'limits:timeout:' + user; + var redisParams = [ + userTimeoutLimitsKey, + 'render', userTimeoutLimit, + 'render_public', userTimeoutLimit + ]; + + redisClient.hmset(redisParams, function (err) { + if (err) { + return callback(err); + } + keysToDelete[userTimeoutLimitsKey] = 5; + return callback(); + }); + }); +} + +function createMapConfig (cartocss) { + return { + version: '1.6.0', + layers: [{ + type: "cartodb", + options: { + sql: [ + 'SELECT', + ' pg_sleep(1),', + ' 1 cartodb_id,', + ' \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' + ].join('\n'), + cartocss: cartocss, + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + }; +} + + + +describe('user timeout limits', function () { + var redisClient = redis.createClient(global.environment.redis.port); + + beforeEach(function() { + keysToDelete = {}; + }); + + afterEach(function (done) { + testHelper.deleteRedisKeys(keysToDelete, done); + }); + + it('layergroup creation works even if test tile is slow', function (done) { + withUserTimeoutRenderLimit(redisClient, 'localhost', 1, function (err) { + if (err) { + return done(err); + } + + var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); + var testClient = new TestClient(mapConfig, 1234); + testClient.getTile(4, 4, 4, {}, function (err, res, tile) { + assert.ok(err, err); + // TODO: check timeout tile + + testClient.drain(done) + }); + }); + }); +}); From 37c970903e5a940d40884488034a402a20a81b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 5 Jul 2017 19:09:14 +0200 Subject: [PATCH 002/128] Avoid uncaught exception when layergroup is not present --- test/support/test-client.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 659d6686..d499e9d5 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -579,8 +579,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { }); }, function finish(err, res, image) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } return callback(err, res, image); } ); From 23aab7a09fa2e7595eadf83bac4d0a9b9b2e4786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 6 Jul 2017 16:23:42 +0200 Subject: [PATCH 003/128] User feature branch of cartodb-redis --- package.json | 2 +- yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 463848c5..dac5756a 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "camshaft": "0.55.6", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", - "cartodb-redis": "0.13.2", + "cartodb-redis": "cartodb/node-cartodb-redis#timeout-limits", "debug": "~2.2.0", "dot": "~1.0.2", "express": "~4.13.3", diff --git a/yarn.lock b/yarn.lock index 8409e6e1..b80967ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -262,9 +262,9 @@ cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" -cartodb-redis@0.13.2: - version "0.13.2" - resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.13.2.tgz#de5214fa5c3ab336c4da978133efa8f908b3691c" +cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: + version "0.13.3" + resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/0c0e5c410243bd4a16e9d94b52821e9d6533eb16" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" @@ -1745,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1770,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: From 32bcf9ca89bf33b72fc00dd90f4a727a74e8f982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 6 Jul 2017 16:24:18 +0200 Subject: [PATCH 004/128] Fix jshint typo --- test/acceptance/user-timeout-limit.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index b595cd0e..cb5fe20e 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -71,11 +71,10 @@ describe('user timeout limits', function () { var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); var testClient = new TestClient(mapConfig, 1234); - testClient.getTile(4, 4, 4, {}, function (err, res, tile) { + testClient.getTile(4, 4, 4, {}, function (err, /* res, tile */) { assert.ok(err, err); // TODO: check timeout tile - - testClient.drain(done) + testClient.drain(done); }); }); }); From ad1506ae976c4976ff6a8b9c6e304d75c61f9b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 6 Jul 2017 16:24:50 +0200 Subject: [PATCH 005/128] Remove empty lines --- test/acceptance/user-timeout-limit.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index cb5fe20e..0c75dce2 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -50,8 +50,6 @@ function createMapConfig (cartocss) { }; } - - describe('user timeout limits', function () { var redisClient = redis.createClient(global.environment.redis.port); From 669707b26c05ab6ef88d35587cd5f0499d345ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 11:56:54 +0200 Subject: [PATCH 006/128] Fix typo --- lib/cartodb/api/user_limits_api.js | 5 ++++- test/acceptance/user-timeout-limit.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 7f244e7d..7e004e31 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -81,10 +81,13 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call }); }, function getUserTimeoutRenderLimits(err, authorized) { + var next = this; + if (err) { return next(err); } - self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, this); + + self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, next); }, function setTilerRenderLimit(err, timeoutRenderLimit) { if (err) { diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index 0c75dce2..c83db965 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -69,7 +69,7 @@ describe('user timeout limits', function () { var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); var testClient = new TestClient(mapConfig, 1234); - testClient.getTile(4, 4, 4, {}, function (err, /* res, tile */) { + testClient.getTile(4, 4, 4, {}, function (err /*, res, tile */) { assert.ok(err, err); // TODO: check timeout tile testClient.drain(done); From 87eb5407a8f9dcfa5937fb88e60c8291d9cb2c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 20:50:31 +0200 Subject: [PATCH 007/128] WIP: implement timeout limit for raster --- test/acceptance/user-timeout-limit.js | 118 ++++++++++++++------------ test/support/test-client.js | 55 ++++++++---- test/support/test_helper.js | 19 ++++- 3 files changed, 120 insertions(+), 72 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index c83db965..1d588f74 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -1,48 +1,25 @@ require('../support/test_helper'); -var assert = require('../support/assert'); -var TestClient = require('../support/test-client'); -var testHelper = require('../support/test_helper'); +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); -var redis = require('redis'); -var keysToDelete; +const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; -function withUserTimeoutRenderLimit(redisClient, user, userTimeoutLimit, callback) { - redisClient.SELECT(5, function(err) { - if (err) { - return callback(err); - } +var pointSleepSql = ` + SELECT + pg_sleep(0.5), + 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, + 1 cartodb_id +`; - var userTimeoutLimitsKey = 'limits:timeout:' + user; - var redisParams = [ - userTimeoutLimitsKey, - 'render', userTimeoutLimit, - 'render_public', userTimeoutLimit - ]; - - redisClient.hmset(redisParams, function (err) { - if (err) { - return callback(err); - } - keysToDelete[userTimeoutLimitsKey] = 5; - return callback(); - }); - }); -} - -function createMapConfig (cartocss) { +function createMapConfig (sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS) { return { version: '1.6.0', layers: [{ - type: "cartodb", + type: 'cartodb', options: { - sql: [ - 'SELECT', - ' pg_sleep(1),', - ' 1 cartodb_id,', - ' \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' - ].join('\n'), - cartocss: cartocss, + sql, + cartocss, cartocss_version: '2.3.0', interactivity: 'cartodb_id' } @@ -51,28 +28,63 @@ function createMapConfig (cartocss) { } describe('user timeout limits', function () { - var redisClient = redis.createClient(global.environment.redis.port); + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; - beforeEach(function() { - keysToDelete = {}; + before(function () { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + }); + + after(function () { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + it('layergroup creation works if test tile is fast but tile request fails if they are slow', function (done) { + var testClient = new TestClient(createMapConfig(), 1234); + + testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + assert.ifError(err); + + testClient.getTile(0, 0, 0, {}, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, function (err) { + assert.ifError(err); + testClient.drain(done); + }); + }); + }); + }); }); - afterEach(function (done) { - testHelper.deleteRedisKeys(keysToDelete, done); - }); + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; - it('layergroup creation works even if test tile is slow', function (done) { - withUserTimeoutRenderLimit(redisClient, 'localhost', 1, function (err) { - if (err) { - return done(err); - } + beforeEach(function() { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + }); - var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); - var testClient = new TestClient(mapConfig, 1234); - testClient.getTile(4, 4, 4, {}, function (err /*, res, tile */) { - assert.ok(err, err); - // TODO: check timeout tile - testClient.drain(done); + afterEach(function() { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + it('layergroup creation works even if test tile is slow', function (done) { + var testClient = new TestClient(createMapConfig(), 1234); + testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + assert.ifError(err); + var params = { + status: 400, + contentType: 'application/json; charset=utf-8' + }; + + testClient.getTile(0, 0, 0, params, function (err, res, tile) { + assert.ifError(err); + + assert.equal(tile.errors[0], 'Render timed out'); + testClient.drain(done); + }); }); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index dbaf466d..80bc9223 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -14,13 +14,13 @@ var helper = require('./test_helper'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); serverOptions.analysis.batch.inlineExecution = true; -var server = new CartodbWindshaft(serverOptions); function TestClient(config, apiKey) { this.mapConfig = isMapConfig(config) ? config : null; this.template = isTemplate(config) ? config : null; this.apiKey = apiKey; this.keysToDelete = {}; + this.server = new CartodbWindshaft(serverOptions); } module.exports = TestClient; @@ -97,7 +97,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -156,7 +156,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -208,7 +208,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -265,7 +265,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call } url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -332,7 +332,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -385,7 +385,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { } url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -441,7 +441,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { params.placeholders = params.placeholders || {}; - assert.response(server, + assert.response(self.server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), method: 'POST', @@ -473,7 +473,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : url; - assert.response(server, + assert.response(self.server, { url: path, method: 'POST', @@ -569,20 +569,22 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; } - assert.response(server, request, expectedResponse, function(res, err) { + if (params.contentType) { + expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + + assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); var obj; if (isPng) { obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } - else if (isMvt) { + } else if (isMvt) { if (res.body) { obj = new mapnik.VectorTile(z, x, y); obj.setDataSync(new Buffer(res.body, 'binary')); } - } - else { + } else { obj = JSON.parse(res.body); } @@ -618,7 +620,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { url += '?' + qs.stringify({api_key: this.apiKey}); } - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -662,7 +664,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -723,7 +725,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { } }; - assert.response(server, request, expectedResponse, function(res, err) { + assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); next(null, res, JSON.parse(res.body)); }); @@ -741,6 +743,10 @@ TestClient.prototype.drain = function(callback) { }; module.exports.getStaticMap = function getStaticMap(templateName, params, callback) { + var self = this; + + self.server = new CartodbWindshaft(serverOptions); + if (!callback) { callback = params; params = null; @@ -771,9 +777,22 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba // this could be removed once named maps are invalidated, otherwise you hits the cache var server = new CartodbWindshaft(serverOptions); - assert.response(server, requestOptions, expectedResponse, function (res, err) { + assert.response(self.server, requestOptions, expectedResponse, function (res, err) { helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() { return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary'))); }); }); }; + +TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) { + const userTimeoutLimitsKey = `limits:timeout:${user}`; + const params = [ + userTimeoutLimitsKey, + 'render', userTimeoutLimit, + 'render_public', userTimeoutLimit + ]; + + this.keysToDelete[userTimeoutLimitsKey] = 5; + + helper.configureMetadata('hmset', params, callback); +} diff --git a/test/support/test_helper.js b/test/support/test_helper.js index de9a6e3c..e8b057c9 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -166,12 +166,29 @@ function rmdirRecursiveSync(dirname) { } } +function configureMetadata(action, params, callback) { + redisClient.SELECT(5, function (err) { + if (err) { + return callback(err); + } + + redisClient[action](params, function (err) { + if (err) { + return callback(err); + } + + return callback(); + }); + }); +} + module.exports = { deleteRedisKeys: deleteRedisKeys, lzma_compress_to_base64: lzma_compress_to_base64, checkNoCache: checkNoCache, checkSurrogateKey: checkSurrogateKey, checkCache: checkCache, - rmdirRecursiveSync: rmdirRecursiveSync + rmdirRecursiveSync: rmdirRecursiveSync, + configureMetadata }; From 8907082a85ef965d2c0de7e4d81592f656273a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 19 Jul 2017 12:24:37 +0200 Subject: [PATCH 008/128] Parse body relying on content-type header --- test/support/test-client.js | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 80bc9223..48ef0eb5 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -575,20 +575,18 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); - var obj; + var body; - if (isPng) { - obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } else if (isMvt) { - if (res.body) { - obj = new mapnik.VectorTile(z, x, y); - obj.setDataSync(new Buffer(res.body, 'binary')); - } - } else { - obj = JSON.parse(res.body); + if (res.headers['content-type'] === 'image/png') { + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + } else if (res.headers['content-type'] === 'application/x-protobuf') { + body = new mapnik.VectorTile(z, x, y); + body.setDataSync(new Buffer(res.body, 'binary')); + } else if (res.headers['content-type'] === 'application/json; charset=utf-8') { + body = JSON.parse(res.body); } - next(null, res, obj); + next(null, res, body); }); }, function finish(err, res, image) { From e9bc0732c0aea7d687864c8c1608497c387dd201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 19 Jul 2017 12:33:57 +0200 Subject: [PATCH 009/128] Use switch statement instead of if ... else if ... else ... --- test/support/test-client.js | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 48ef0eb5..9d1d1d7e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -575,15 +575,22 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); - var body; - if (res.headers['content-type'] === 'image/png') { - body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } else if (res.headers['content-type'] === 'application/x-protobuf') { - body = new mapnik.VectorTile(z, x, y); - body.setDataSync(new Buffer(res.body, 'binary')); - } else if (res.headers['content-type'] === 'application/json; charset=utf-8') { - body = JSON.parse(res.body); + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/x-protobuf': + body = new mapnik.VectorTile(z, x, y); + body.setDataSync(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; } next(null, res, body); From eee4fc815ebad60a948e0112e2641691987b8c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:11:56 +0200 Subject: [PATCH 010/128] Do not expose database error details --- lib/cartodb/models/dataview/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index f8eb1027..7c3dbe6f 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -9,9 +9,9 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { var self = this; this.sql(psql, override, function(err, query) { psql.query(query, function(err, result) { - if (err) { - return callback(err, result); + var error = new Error(err.message); + return callback(error, result); } result = self.format(result, override); From 444595d49d41efcc0b74517357b7048149727bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:14:04 +0200 Subject: [PATCH 011/128] Rely on windshaft's repo branch --- package.json | 2 +- yarn.lock | 92 ++++++++++++++++++++++++++-------------------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/package.json b/package.json index 19c00700..093a6948 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.2.2", + "windshaft": "cartodb/windshaft#response-timeout-limit", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index c6a468fa..14962708 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,7 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -239,6 +231,14 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -1741,32 +1741,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.81.0: +request@2.x, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1793,6 +1768,31 @@ request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -2103,17 +2103,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3": - version "2.3.1-cdb3" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23" +tilelive-bridge@cartodb/tilelive-bridge#query-timeout: + version "2.3.1-cdb2" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/38d5d82355aa9eef68c29b7d7a33b60a3ae394f7" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#custom-timeout: version "0.6.18-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/d9bfe27a4d5120be92e71337474135e2877e9cd4" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2274,9 +2274,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.2.2: +windshaft@cartodb/windshaft#response-timeout-limit: version "3.2.2" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.2.tgz#7afb9d8fd8bba1bf02d39c06e8bbe5a451aad953" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/21ebe5f69b5b6215ec2f93088d8eee436f6b81d5" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2293,8 +2293,8 @@ windshaft@3.2.2: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb3 - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2 + tilelive-bridge cartodb/tilelive-bridge#query-timeout + tilelive-mapnik cartodb/tilelive-mapnik#custom-timeout torque.js "~2.11.0" underscore "~1.6.0" From 526e850f26b364be02a316401b05bc6a153b2b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:15:43 +0200 Subject: [PATCH 012/128] Add method to set statements timeout for user's role and database --- test/support/test-client.js | 73 ++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 9d1d1d7e..411cdf3b 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -3,7 +3,8 @@ var qs = require('querystring'); var step = require('step'); var urlParser = require('url'); - +var PSQL = require('cartodb-psql'); +var _ = require('underscore'); var mapnik = require('windshaft').mapnik; var LayergroupToken = require('./layergroup-token'); @@ -72,6 +73,34 @@ module.exports.CARTOCSS = { ' line-width: 0.5;', ' line-opacity: 1;', '}' + ].join('\n'), + + TORQUE: [ + 'Map {', + ' -torque-frame-count: 256;', + ' -torque-animation-duration: 30;', + ' -torque-time-attribute: "cartodb_id";', + ' -torque-aggregation-function: "count(1)";', + ' -torque-resolution: 4;', + ' -torque-data-aggregation: linear;', + '}', + '#layer {', + ' marker-width: 7;', + ' marker-fill: #FFB927;', + ' marker-fill-opacity: 0.9;', + ' marker-line-width: 1;', + ' marker-line-color: #FFF;', + ' marker-line-opacity: 1;', + ' comp-op: lighter;', + '}', + '#layer[frame-offset=1] {', + ' marker-width: 9;', + ' marker-fill-opacity: 0.45;', + '}', + '#layer[frame-offset=2] {', + ' marker-width: 11;', + ' marker-fill-opacity: 0.225;', + '}' ].join('\n') }; @@ -800,4 +829,44 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi this.keysToDelete[userTimeoutLimitsKey] = 5; helper.configureMetadata('hmset', params, callback); -} +}; + +TestClient.prototype.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { + const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; + const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + + const psql = new PSQL({ + user: 'postgres', + dbname: dbname, + host: global.environment.postgres.host, + port: global.environment.postgres.port + }); + + step( + function setTimeoutToUserRole (err, params) { + const next = this; + + const timeoutQuery = `ALTER ROLE \"${role}\" SET statement_timeout to ${userTimeoutLimit}`; + psql.query(timeoutQuery, function (err) { + if (err) { + return next(err); + } + next(null, params, psql); + }); + }, + function setTimeoutToDatabase (err, params, psql) { + assert.ifError(err); + + const timeoutQuery = `ALTER DATABASE \"${dbname}\" SET statement_timeout to ${userTimeoutLimit}`; + + psql.query(timeoutQuery, this); + }, + function finish (err) { + if (err) { + return callback(err); + } + + callback(); + } + ); +}; From 788b2f06830d093554edbed6d681bc4416ab9d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:16:37 +0200 Subject: [PATCH 013/128] Implement test to validate response limits work as expected --- test/acceptance/user-timeout-limit.js | 324 ++++++++++++++++++++++---- 1 file changed, 278 insertions(+), 46 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index 1d588f74..6d30785a 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -5,87 +5,319 @@ const TestClient = require('../support/test-client'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; -var pointSleepSql = ` +const pointSleepSql = ` SELECT pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id `; -function createMapConfig (sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS) { - return { - version: '1.6.0', - layers: [{ - type: 'cartodb', - options: { - sql, - cartocss, - cartocss_version: '2.3.0', - interactivity: 'cartodb_id' +const validationPointSleepSql = ` + SELECT + pg_sleep(0.5), + 'SRID=3857;POINT(-180 90)'::geometry the_geom_webmercator, + 1 cartodb_id +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = pointSleepSql, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: "a0" + }, + cartocss, + cartocss_version, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql } - }] - }; -} + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); -describe('user timeout limits', function () { - describe('with onTileErrorStrategy ENABLED', function () { - let onTileErrorStrategy; - - before(function () { - onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; - global.environment.enabledFeatures.onTileErrorStrategy = true; +describe('user timeout limit', function () { + describe.skip('map instantiation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); }); - after(function () { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + afterEach(function (done) { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); - it('layergroup creation works if test tile is fast but tile request fails if they are slow', function (done) { - var testClient = new TestClient(createMapConfig(), 1234); + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; - testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.ifError(err); - testClient.getTile(0, 0, 0, {}, function (err, res, tile) { - assert.ifError(err); - - assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, function (err) { - assert.ifError(err); - testClient.drain(done); - }); + assert.deepEqual(timeoutError, { + errors: ['Render timed out'], + errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] }); + + done(); }); }); }); - describe('with onTileErrorStrategy DISABLED', function() { - var onTileErrorStrategy; - - beforeEach(function() { - onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; - global.environment.enabledFeatures.onTileErrorStrategy = false; + describe('user torque timeout limits', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); }); - afterEach(function() { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); - it('layergroup creation works even if test tile is slow', function (done) { - var testClient = new TestClient(createMapConfig(), 1234); - testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.ifError(err); + + assert.deepEqual(timeoutError, { + errors: ["TorqueRenderer: canceling statement due to statement timeout"], + errors_with_context: [{ + "type": "layer", + "message": "TorqueRenderer: canceling statement due to statement timeout", + "layer": { "id": "torque-layer0", "index": 0, "type": "torque" } + }] + }); + + done(); + }); + }); + }); + + describe('dataview', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but dataview request fails due to statement timeout', function (done) { + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getDataview('count', params, (err, dataview) => { + assert.ifError(err); + + assert.deepEqual(dataview, { + errors: ['canceling statement due to statement timeout'], + errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] + }); + + done(); + }); + }); + }); + + + describe('raster', function () { + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function () { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + }); + + afterEach(function () { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + + it('layergroup creation works but tile request fails due to render timeout', function (done) { + this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + }); + + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; + + beforeEach(function() { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + }); + + afterEach(function() { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works even if render tile is slow', function (done) { var params = { status: 400, contentType: 'application/json; charset=utf-8' }; - testClient.getTile(0, 0, 0, params, function (err, res, tile) { + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); assert.equal(tile.errors[0], 'Render timed out'); - testClient.drain(done); + done(); }); }); }); }); + + describe('user vector timeout limits', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + status: 400 + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['Render timed out'], + errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + }); + + done(); + }); + }); + }); + + describe('user interativity timeout limits', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) { + const params = { + layers: 'mapnik', + format: 'grid.json', + status: 400 + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['Render timed out'], + errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + }); + + done(); + }); + }); + }); }); + From 058f19ab3646b3dcd7c02f7996b159fc123b590e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 26 Jul 2017 13:25:37 +0200 Subject: [PATCH 014/128] Fix skipped test --- test/acceptance/user-timeout-limit.js | 33 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index 6d30785a..f4a3877e 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -9,14 +9,17 @@ const pointSleepSql = ` SELECT pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, - 1 cartodb_id + 1 cartodb_id, + 2 value `; +// during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout const validationPointSleepSql = ` SELECT - pg_sleep(0.5), - 'SRID=3857;POINT(-180 90)'::geometry the_geom_webmercator, - 1 cartodb_id + pg_sleep(1), + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 value `; const createMapConfig = ({ @@ -26,18 +29,20 @@ const createMapConfig = ({ cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', interactivity = 'cartodb_id', - countBy = 'cartodb_id' + countBy = 'cartodb_id', + attributes = { id: 'cartodb_id', columns: ['value'] }, } = {}) => ({ version, layers: [{ type, options: { source: { - id: "a0" + id: 'a0' }, cartocss, cartocss_version, - interactivity + interactivity, + attributes } }], analyses: [ @@ -64,7 +69,7 @@ const createMapConfig = ({ }); describe('user timeout limit', function () { - describe.skip('map instantiation', function () { + describe('map instantiation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); @@ -92,8 +97,16 @@ describe('user timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ['Render timed out'], - errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + errors: ["Render timed out"], + errors_with_context: [{ + type: "layer", + message: "Render timed out", + layer: { + id: "layer0", + index: 0, + type: "mapnik" + } + }] }); done(); From 768d06c582098ecfd4b6de9430856c57dd6fa7a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 26 Jul 2017 18:35:40 +0200 Subject: [PATCH 015/128] Assert layer params is defined but mapnik layers --- test/support/test-client.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 411cdf3b..961914d7 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -16,6 +16,14 @@ var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); serverOptions.analysis.batch.inlineExecution = true; +const MAPNIK_SUPPORTED_FORMATS = { + 'png': true, + 'png32': true, + 'grid.json': true, + 'geojson': true, + 'mvt': true +} + function TestClient(config, apiKey) { this.mapConfig = isMapConfig(config) ? config : null; this.template = isTemplate(config) ? config : null; @@ -543,6 +551,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { var format = params.format || 'png'; + if (layers === undefined && !MAPNIK_SUPPORTED_FORMATS[format]) { + throw new Error(`Missing layer filter while fetching ${format} tile, review params argument`); + } + url += [z,x,y].join('/'); url += '.' + format; From 522fc79d71dfa4f70baee5a9ca53713eb791ff3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 12:46:38 +0200 Subject: [PATCH 016/128] Do not remove redis' keys if layergroup was not created --- test/support/test-client.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 961914d7..c78d6e04 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -441,9 +441,16 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }, function finish(err, dataview) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - return callback(err, dataview); + if (err) { + return callback(err); + } + + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(null, dataview); } ); }; From 9e491e7e9a44d2134e2b896fc47b67062c37bb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 16:29:36 +0200 Subject: [PATCH 017/128] Fix tests names --- test/acceptance/user-timeout-limit.js | 4 ++-- test/support/test-client.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index f4a3877e..d3de7164 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -272,7 +272,7 @@ describe('user timeout limit', function () { }); }); - describe('user vector timeout limits', function () { + describe('vector', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); @@ -302,7 +302,7 @@ describe('user timeout limit', function () { }); }); - describe('user interativity timeout limits', function () { + describe('interativity', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); diff --git a/test/support/test-client.js b/test/support/test-client.js index c78d6e04..80869152 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -850,7 +850,7 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.prototype.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { +TestClient.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) From eba97a41e5b912c4a87793583aa1d306a043456f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 16:30:57 +0200 Subject: [PATCH 018/128] Going green, solves issue with role timeout in database --- test/acceptance/user-timeout-limit.js | 108 ++++++++++++-------------- 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index d3de7164..dca4ece7 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -7,10 +7,10 @@ const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.pn const pointSleepSql = ` SELECT - pg_sleep(0.5), + pg_sleep(1), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, - 2 value + 2 val `; // during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout @@ -19,7 +19,7 @@ const validationPointSleepSql = ` pg_sleep(1), ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, 1 cartodb_id, - 2 value + 2 val `; const createMapConfig = ({ @@ -30,7 +30,7 @@ const createMapConfig = ({ cartocss_version = '2.3.0', interactivity = 'cartodb_id', countBy = 'cartodb_id', - attributes = { id: 'cartodb_id', columns: ['value'] }, + attributes } = {}) => ({ version, layers: [{ @@ -68,7 +68,15 @@ const createMapConfig = ({ } }); -describe('user timeout limit', function () { +describe.only('user timeout limit', function () { + before(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 900, done); + }); + + after(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + describe('map instantiation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); @@ -114,64 +122,14 @@ describe('user timeout limit', function () { }); }); - describe('user torque timeout limits', function () { - beforeEach(function (done) { - const mapconfig = createMapConfig({ - type: 'torque', - cartocss: TestClient.CARTOCSS.TORQUE - }); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); - }); - - afterEach(function (done) { - this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); - } - this.testClient.drain(done); - }); - }); - - it('layergroup creation fails due to statement timeout', function (done) { - const expectedResponse = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { - assert.ifError(err); - - assert.deepEqual(timeoutError, { - errors: ["TorqueRenderer: canceling statement due to statement timeout"], - errors_with_context: [{ - "type": "layer", - "message": "TorqueRenderer: canceling statement due to statement timeout", - "layer": { "id": "torque-layer0", "index": 0, "type": "torque" } - }] - }); - - done(); - }); - }); - }); - describe('dataview', function () { - beforeEach(function (done) { + beforeEach(function () { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); }); afterEach(function (done) { - this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); - } - this.testClient.drain(done); - }); + this.testClient.drain(done); }); it('layergroup creation works but dataview request fails due to statement timeout', function (done) { @@ -197,6 +155,42 @@ describe('user timeout limit', function () { }); }); + describe('torque', function () { + beforeEach(function () { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: ["TorqueRenderer: canceling statement due to statement timeout"], + errors_with_context: [{ + type: "layer", + message: "TorqueRenderer: canceling statement due to statement timeout", + layer: { id: 'torque-layer0', index: 0, type: "torque" } + }] + }); + + done(); + }); + }); + }); + describe('raster', function () { describe('with onTileErrorStrategy ENABLED', function () { From 664a4e673a1aec5d561d7a156a1c3794382afbd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 17:08:29 +0200 Subject: [PATCH 019/128] Add missing after-each hooks and merged duplicates --- test/acceptance/user-timeout-limit.js | 50 +++++++++++++++++---------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index dca4ece7..87c07b76 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -68,7 +68,7 @@ const createMapConfig = ({ } }); -describe.only('user timeout limit', function () { +describe('user timeout limit', function () { before(function (done) { TestClient.setUserDatabaseTimeoutLimit('localhost', 900, done); }); @@ -196,23 +196,24 @@ describe.only('user timeout limit', function () { describe('with onTileErrorStrategy ENABLED', function () { let onTileErrorStrategy; - beforeEach(function () { + beforeEach(function (done) { onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; global.environment.enabledFeatures.onTileErrorStrategy = true; - }); - afterEach(function () { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - }); - - beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); }); afterEach(function (done) { - this.testClient.drain(done); + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); @@ -231,23 +232,24 @@ describe.only('user timeout limit', function () { describe('with onTileErrorStrategy DISABLED', function() { var onTileErrorStrategy; - beforeEach(function() { + beforeEach(function (done) { onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; global.environment.enabledFeatures.onTileErrorStrategy = false; - }); - afterEach(function() { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - }); - - beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); }); afterEach(function (done) { - this.testClient.drain(done); + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works even if render tile is slow', function (done) { @@ -274,7 +276,12 @@ describe.only('user timeout limit', function () { }); afterEach(function (done) { - this.testClient.drain(done); + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works but vector tile request fails due to render timeout', function (done) { @@ -304,7 +311,12 @@ describe.only('user timeout limit', function () { }); afterEach(function (done) { - this.testClient.drain(done); + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) { From 33e77a42f21f3e5e3243a05f122ee91ea9c5a7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 18:50:27 +0200 Subject: [PATCH 020/128] Separate user timeout suites between render and database --- .../acceptance/user-database-timeout-limit.js | 133 ++++++++++++++++++ ...-limit.js => user-render-timeout-limit.js} | 82 +---------- test/support/test-client.js | 36 ++--- 3 files changed, 147 insertions(+), 104 deletions(-) create mode 100644 test/acceptance/user-database-timeout-limit.js rename test/acceptance/{user-timeout-limit.js => user-render-timeout-limit.js} (75%) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js new file mode 100644 index 00000000..32976b37 --- /dev/null +++ b/test/acceptance/user-database-timeout-limit.js @@ -0,0 +1,133 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); + +const pointSleepSql = ` + SELECT + pg_sleep(1), + 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = pointSleepSql, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); + +describe('user database timeout limit', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + describe('dataview', function () { + beforeEach(function () { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works but dataview request fails due to statement timeout', function (done) { + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getDataview('count', params, (err, dataview) => { + assert.ifError(err); + + assert.deepEqual(dataview, { + errors: ['canceling statement due to statement timeout'], + errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] + }); + + done(); + }); + }); + }); + + describe('torque', function () { + beforeEach(function () { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: ["TorqueRenderer: canceling statement due to statement timeout"], + errors_with_context: [{ + type: "layer", + message: "TorqueRenderer: canceling statement due to statement timeout", + layer: { id: 'torque-layer0', index: 0, type: "torque" } + }] + }); + + done(); + }); + }); + }); +}); diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js similarity index 75% rename from test/acceptance/user-timeout-limit.js rename to test/acceptance/user-render-timeout-limit.js index 87c07b76..8d7e9a6e 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -68,16 +68,8 @@ const createMapConfig = ({ } }); -describe('user timeout limit', function () { - before(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 900, done); - }); - - after(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); - }); - - describe('map instantiation', function () { +describe('user render timeout limit', function () { + describe('map instantiation => validation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); @@ -122,76 +114,6 @@ describe('user timeout limit', function () { }); }); - describe('dataview', function () { - beforeEach(function () { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - }); - - afterEach(function (done) { - this.testClient.drain(done); - }); - - it('layergroup creation works but dataview request fails due to statement timeout', function (done) { - const params = { - response: { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - } - }; - - this.testClient.getDataview('count', params, (err, dataview) => { - assert.ifError(err); - - assert.deepEqual(dataview, { - errors: ['canceling statement due to statement timeout'], - errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] - }); - - done(); - }); - }); - }); - - describe('torque', function () { - beforeEach(function () { - const mapconfig = createMapConfig({ - type: 'torque', - cartocss: TestClient.CARTOCSS.TORQUE - }); - this.testClient = new TestClient(mapconfig, 1234); - }); - - afterEach(function (done) { - this.testClient.drain(done); - }); - - it('layergroup creation fails due to statement timeout', function (done) { - const expectedResponse = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { - assert.deepEqual(timeoutError, { - errors: ["TorqueRenderer: canceling statement due to statement timeout"], - errors_with_context: [{ - type: "layer", - message: "TorqueRenderer: canceling statement due to statement timeout", - layer: { id: 'torque-layer0', index: 0, type: "torque" } - }] - }); - - done(); - }); - }); - }); - - describe('raster', function () { describe('with onTileErrorStrategy ENABLED', function () { let onTileErrorStrategy; diff --git a/test/support/test-client.js b/test/support/test-client.js index 80869152..d03fef2e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -850,9 +850,10 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { +TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + const publicUser = global.environment.postgres.user; const psql = new PSQL({ user: 'postgres', @@ -862,30 +863,17 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callb }); step( - function setTimeoutToUserRole (err, params) { - const next = this; + function configureTimeouts () { + const timeoutSQLs = [ + `ALTER ROLE \"${publicUser}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER ROLE \"${role}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER DATABASE \"${dbname}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` + ]; - const timeoutQuery = `ALTER ROLE \"${role}\" SET statement_timeout to ${userTimeoutLimit}`; - psql.query(timeoutQuery, function (err) { - if (err) { - return next(err); - } - next(null, params, psql); - }); + const group = this.group(); + + timeoutSQLs.forEach(sql => psql.query(sql, group())); }, - function setTimeoutToDatabase (err, params, psql) { - assert.ifError(err); - - const timeoutQuery = `ALTER DATABASE \"${dbname}\" SET statement_timeout to ${userTimeoutLimit}`; - - psql.query(timeoutQuery, this); - }, - function finish (err) { - if (err) { - return callback(err); - } - - callback(); - } + callback ); }; From b023a155b7782faa52aae6b430f88979f3d28b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 13:21:17 +0200 Subject: [PATCH 021/128] Be more accurate with timeouts and pg_sleep --- test/acceptance/user-database-timeout-limit.js | 4 ++-- test/acceptance/user-render-timeout-limit.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 32976b37..c59b26df 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -5,7 +5,7 @@ const TestClient = require('../support/test-client'); const pointSleepSql = ` SELECT - pg_sleep(1), + pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, 2 val @@ -55,7 +55,7 @@ const createMapConfig = ({ describe('user database timeout limit', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 50, done); + TestClient.setUserDatabaseTimeoutLimit('localhost', 300, done); }); afterEach(function (done) { diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 8d7e9a6e..0119a1b9 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -16,7 +16,7 @@ const pointSleepSql = ` // during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout const validationPointSleepSql = ` SELECT - pg_sleep(1), + pg_sleep(0.5), ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, 1 cartodb_id, 2 val From 815eac5a48987a521da7e2e215f54f3ab7727b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 13:22:16 +0200 Subject: [PATCH 022/128] Add hook to refresh all connections in the pool --- test/support/test_helper.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/support/test_helper.js b/test/support/test_helper.js index e8b057c9..fc9fcb34 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -14,6 +14,7 @@ var lzmaWorker = new LZMA(); var redis = require('redis'); var nock = require('nock'); var log4js = require('log4js'); +var pg = require('pg'); // set environment specific variables global.environment = require(__dirname + '/../../config/environments/test'); @@ -127,6 +128,12 @@ afterEach(function(done) { }); }); + +afterEach(function () { + // TODO: this method will be replaced by psql.end + pg.end(); +}); + function deleteRedisKeys(keysToDelete, callback) { if (Object.keys(keysToDelete).length === 0) { From cca570e832ea917eae7532467229f7fa0295c7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 13:23:13 +0200 Subject: [PATCH 023/128] Uncomment DB and role configuration --- test/support/test-client.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index d03fef2e..1d66d2b6 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -852,8 +852,9 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; - const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) - const publicUser = global.environment.postgres.user; + const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) + const publicuser = global.environment.postgres.user; const psql = new PSQL({ user: 'postgres', @@ -865,9 +866,9 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) step( function configureTimeouts () { const timeoutSQLs = [ - `ALTER ROLE \"${publicUser}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, - `ALTER ROLE \"${role}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, - `ALTER DATABASE \"${dbname}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` + // `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + // `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` ]; const group = this.group(); From c3e137bb0029538c8bd4913f010c2057477884d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 16:02:54 +0200 Subject: [PATCH 024/128] Update dependency --- yarn.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index 14962708..d7c06ac0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2113,7 +2113,7 @@ tilelive-bridge@cartodb/tilelive-bridge#query-timeout: tilelive-mapnik@cartodb/tilelive-mapnik#custom-timeout: version "0.6.18-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/d9bfe27a4d5120be92e71337474135e2877e9cd4" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/f9042e5eab92e26fb9e2a6ace65af92cc176a218" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" From 6936107b683d7a9441a07870a0ca8c96f80bc66a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 16:04:11 +0200 Subject: [PATCH 025/128] Adjust pg_sleep to timeout --- test/acceptance/user-render-timeout-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 0119a1b9..8c167d4f 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -7,7 +7,7 @@ const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.pn const pointSleepSql = ` SELECT - pg_sleep(1), + pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, 2 val From f2e043b0631d97d7bcbd65cc5a11ce8d095a2790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 17:56:58 +0200 Subject: [PATCH 026/128] Do not expose database error info --- lib/cartodb/controllers/base.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 7cdd0713..30883e04 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -229,6 +229,18 @@ function stripConnectionInfo(message) { .replace(/is the server.*encountered/im, 'encountered'); } +var ERROR_INFO_TO_EXOSE = { + message: true, + layer: true, + type: true, + analysis: true, + subtype: true +}; + +function shouldBeExposed (prop) { + return !!ERROR_INFO_TO_EXOSE[prop]; +} + function errorMessage(err) { // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 var message = (_.isString(err) ? err : err.message) || 'Unknown error'; @@ -247,7 +259,7 @@ function errorMessageWithContext(err) { for (var prop in err) { // type & message are properties from Error's prototype and will be skipped - if (err.hasOwnProperty(prop)) { + if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) { error[prop] = err[prop]; } } From 398369a5c7ef301cb11975059391e7c9731cf90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 17:58:33 +0200 Subject: [PATCH 027/128] Do not rely on expected defaults headers --- test/acceptance/mvt.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index b1ea4455..b5a59cd8 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -24,14 +24,24 @@ describe('mvt', function () { desc: 'should get empty mvt with code 204 (no content)', coords: { z: 0, x: 0, y: 0 }, format: 'mvt', - status: 204, + response: { + status: 204, + headers: { + 'Content-Type': undefined + } + }, mapConfig: createMapConfig(TestClient.SQL.EMPTY) }, { desc: 'should get mvt tile with code 200 (ok)', coords: { z: 0, x: 0, y: 0 }, format: 'mvt', - status: 200, + response: { + status: 200, + headers: { + 'Content-Type': 'application/x-protobuf' + } + }, mapConfig: createMapConfig() } ]; @@ -40,12 +50,12 @@ describe('mvt', function () { it(test.desc, done => { const testClient = new TestClient(test.mapConfig, 1234); const { z, x, y } = test.coords; - const { format, status } = test; + const { format, response } = test; - testClient.getTile(z, x, y, { format, status }, (err, res) => { + testClient.getTile(z, x, y, { format, response }, (err, res) => { assert.ifError(err); - assert.equal(res.statusCode, test.status); + assert.equal(res.statusCode, test.response.status); testClient.drain(done); }); }); From 64f19b65ec411b4d7f4e13c15513b85a17b8dbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:01:19 +0200 Subject: [PATCH 028/128] Remove attributes param --- test/acceptance/user-render-timeout-limit.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 8c167d4f..a44c8793 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -29,8 +29,7 @@ const createMapConfig = ({ cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', interactivity = 'cartodb_id', - countBy = 'cartodb_id', - attributes + countBy = 'cartodb_id' } = {}) => ({ version, layers: [{ @@ -41,8 +40,7 @@ const createMapConfig = ({ }, cartocss, cartocss_version, - interactivity, - attributes + interactivity } }], analyses: [ From 664db4b5cfefbffc9886a7cfd16ac25e44284543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:07:44 +0200 Subject: [PATCH 029/128] Pass proper param to check content-type and status --- test/acceptance/user-render-timeout-limit.js | 25 +++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index a44c8793..cd5b03a7 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -136,7 +136,6 @@ describe('user render timeout limit', function () { }); }); - it('layergroup creation works but tile request fails due to render timeout', function (done) { this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { assert.ifError(err); @@ -172,10 +171,14 @@ describe('user render timeout limit', function () { }); }); - it('layergroup creation works even if render tile is slow', function (done) { + it('layergroup creation works and render tile fails', function (done) { var params = { - status: 400, - contentType: 'application/json; charset=utf-8' + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { @@ -207,7 +210,12 @@ describe('user render timeout limit', function () { it('layergroup creation works but vector tile request fails due to render timeout', function (done) { const params = { format: 'mvt', - status: 400 + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { @@ -243,7 +251,12 @@ describe('user render timeout limit', function () { const params = { layers: 'mapnik', format: 'grid.json', - status: 400 + response: { + status: 400, + headers: { + 'Content-Type': 'application/x-protobuf' + } + } }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { From 435d902e45bf3146fa13d4662eb7a230267dcff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:10:14 +0200 Subject: [PATCH 030/128] Expose function to clean all database connections in the pool --- test/support/test_helper.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/support/test_helper.js b/test/support/test_helper.js index fc9fcb34..49c6c1ee 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -128,11 +128,10 @@ afterEach(function(done) { }); }); - -afterEach(function () { +function cleanPGPoolConnections () { // TODO: this method will be replaced by psql.end pg.end(); -}); +} function deleteRedisKeys(keysToDelete, callback) { @@ -196,6 +195,7 @@ module.exports = { checkSurrogateKey: checkSurrogateKey, checkCache: checkCache, rmdirRecursiveSync: rmdirRecursiveSync, - configureMetadata + configureMetadata, + cleanPGPoolConnections }; From aa6d01f151a5fb614ce8ad3980035f10c6a1eed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:12:33 +0200 Subject: [PATCH 031/128] Add timeout test for attributes --- test/support/test-client.js | 104 ++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 1d66d2b6..8b6ab3e8 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -791,6 +791,110 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { ); }; +TestClient.prototype.getAttributes = function(params, callback) { + var self = this; + + if (!Number.isFinite(params.featureId)) { + throw new Error('featureId param must be a number') + } + + if (!Number.isFinite(params.layer)) { + throw new Error('layer param must be a number') + } + + var url = '/api/v1/map'; + + if (this.apiKey) { + url += '?' + qs.stringify({ api_key: this.apiKey }); + } + + var layergroupid; + + if (params.layergroupid) { + layergroupid = params.layergroupid + } + + step( + function createLayergroup() { + var next = this; + + if (layergroupid) { + return next(null, layergroupid); + } + + assert.response(self.server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + var parsedBody = JSON.parse(res.body); + + return next(null, parsedBody.layergroupid); + } + ); + }, + function getAttributes(err, _layergroupid) { + assert.ifError(err); + + var next = this; + + layergroupid = _layergroupid; + + url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`; + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }; + + var expectedResponse = params.response || { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + assert.response(self.server, request, expectedResponse, function (res, err) { + assert.ifError(err); + + var attributes = JSON.parse(res.body); + + next(null, res, attributes); + }); + }, + function finish(err, res, attributes) { + if (layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(err, res, attributes); + } + ); +}; + TestClient.prototype.drain = function(callback) { helper.deleteRedisKeys(this.keysToDelete, callback); }; From 04da57fe0cfc7b566a0315edd1c514a9499751de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:14:32 +0200 Subject: [PATCH 032/128] Do not create layergroup if it is already provided --- test/support/test-client.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 8b6ab3e8..34d04200 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -471,6 +471,11 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } var layergroupId; + + if (params.layergroupid) { + layergroupId = params.layergroupid + } + step( function createTemplate () { var next = this; @@ -512,6 +517,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { function createLayergroup(err, templateId) { var next = this; + if (layergroupId) { + return next(null, layergroupId); + } + var data = templateId ? params.placeholders : self.mapConfig var path = templateId ? urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : From f079c24554bbfc8c5d10e525770d4c7d9867f944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:22:13 +0200 Subject: [PATCH 033/128] Use parsed body variable --- test/support/test-client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 34d04200..2f322770 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -707,7 +707,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { return callback(err); } - return callback(null, JSON.parse(res.body)); + return callback(null, parsedBody); } ); }; From 9b5482489e2e9261a6fe216d04b5aa1e21136b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:23:17 +0200 Subject: [PATCH 034/128] Fix content-type defaults --- test/support/test-client.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 2f322770..04a701b4 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -586,32 +586,30 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }; - var expectedResponse = { - status: params.status || 200, + var expectedResponse = Object.assign({}, { + status: 200, headers: { - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'image/png' } - }; + }, params.response); + var isPng = format.match(/png$/); if (isPng) { request.encoding = 'binary'; - expectedResponse.headers['Content-Type'] = 'image/png'; } var isMvt = format.match(/mvt$/); if (isMvt) { request.encoding = 'binary'; - if (expectedResponse.status === 200) { expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; - } else if (expectedResponse.status === 204) { - expectedResponse.headers['Content-Type'] = undefined; } } + var isGeojson = format.match(/geojson$/); if (isGeojson) { From ac7509b01a0029c3728871502fe90ec17bb8fdfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:24:42 +0200 Subject: [PATCH 035/128] Expose function to clean up database connections --- test/support/test-client.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 04a701b4..e35824d7 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -967,6 +967,9 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) const publicuser = global.environment.postgres.user; + // we need to guarantee all new connections have the new settings + helper.cleanPGPoolConnections() + const psql = new PSQL({ user: 'postgres', dbname: dbname, @@ -989,3 +992,8 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) callback ); }; + +TestClient.cleanDatabaseConnections = function () { + helper.cleanPGPoolConnections() +} + From 227c2b336bcfc817f0cfb3ace69ac373ffe4e1c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:25:34 +0200 Subject: [PATCH 036/128] Uncomment database timeout configuration --- test/support/test-client.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index e35824d7..750a9f69 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -980,9 +980,9 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) step( function configureTimeouts () { const timeoutSQLs = [ - // `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, `ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, - // `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` + `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` ]; const group = this.group(); From b1114fc6063fdda92e3a554b5ac0f54f176d9439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:26:45 +0200 Subject: [PATCH 037/128] Add timeout test for toque.json tiles --- .../acceptance/user-database-timeout-limit.js | 272 +++++++++++++++--- 1 file changed, 236 insertions(+), 36 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index c59b26df..da9183c7 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -5,7 +5,7 @@ const TestClient = require('../support/test-client'); const pointSleepSql = ` SELECT - pg_sleep(0.5), + pg_sleep(0.3), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, 2 val @@ -17,7 +17,8 @@ const createMapConfig = ({ sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', - countBy = 'cartodb_id' + countBy = 'cartodb_id', + attributes } = {}) => ({ version, layers: [{ @@ -27,7 +28,8 @@ const createMapConfig = ({ id: 'a0' }, cartocss, - cartocss_version + cartocss_version, + attributes } }], analyses: [ @@ -54,15 +56,15 @@ const createMapConfig = ({ }); describe('user database timeout limit', function () { - beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 300, done); - }); - - afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); - }); - describe('dataview', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + beforeEach(function () { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); @@ -95,38 +97,236 @@ describe('user database timeout limit', function () { }); }); - describe('torque', function () { - beforeEach(function () { - const mapconfig = createMapConfig({ - type: 'torque', - cartocss: TestClient.CARTOCSS.TORQUE + describe('torque:', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors_with_context: [{ + type: 'layer', + message: 'TorqueRenderer: canceling statement due to statement timeout', + layer: { id: 'torque-layer0', index: 0, type: 'torque' } + }] + }); + + done(); + }); }); - this.testClient = new TestClient(mapconfig, 1234); }); - afterEach(function (done) { - this.testClient.drain(done); - }); + describe('fetching "torque.json" tile', function () { + before(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; - it('layergroup creation fails due to statement timeout', function (done) { - const expectedResponse = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { - assert.deepEqual(timeoutError, { - errors: ["TorqueRenderer: canceling statement due to statement timeout"], - errors_with_context: [{ - type: "layer", - message: "TorqueRenderer: canceling statement due to statement timeout", - layer: { id: 'torque-layer0', index: 0, type: "torque" } - }] + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); }); - done(); + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'torque.json', + layers: [ 0 ], + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { + assert.ifError(err); + + assert.deepEqual(attributes, { + errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors_with_context: [{ + type: 'unknown', + message: 'TorqueRenderer: canceling statement due to statement timeout', + }] + }); + + done(); + }); + }); + }); + }); + }); + + describe('attributes:', function () { + describe('while validating in map instatiation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + attributes: { + id: 'cartodb_id', + columns: [ 'val' ] + } + }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'canceling statement due to statement timeout' ], + errors_with_context: [{ + type: 'layer', + message: 'canceling statement due to statement timeout', + layer: { + id: 'layer0', + index: 0, + type: 'mapnik' + } + }] + }); + + done(); + }); + }); + }); + + describe('fetching by feature id', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + attributes: { + id: 'cartodb_id', + columns: [ 'val' ] + } + }); + + this.testClient = new TestClient(mapconfig, 1234); + + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + featureId: 1, + layer: 0, + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getAttributes(params, (err, res, attributes) => { + assert.ifError(err); + + assert.deepEqual(attributes, { + errors: ['canceling statement due to statement timeout'], + errors_with_context: [{ + type: 'unknown', + message: 'canceling statement due to statement timeout' + }] + }); + + done(); + }); + }); }); }); }); From 49d2f513c6c609e692c86383f7bfe1be5e5eb8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:51:23 +0200 Subject: [PATCH 038/128] Fix typo --- lib/cartodb/controllers/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 30883e04..b8b73569 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -229,7 +229,7 @@ function stripConnectionInfo(message) { .replace(/is the server.*encountered/im, 'encountered'); } -var ERROR_INFO_TO_EXOSE = { +var ERROR_INFO_TO_EXPOSE = { message: true, layer: true, type: true, @@ -238,7 +238,7 @@ var ERROR_INFO_TO_EXOSE = { }; function shouldBeExposed (prop) { - return !!ERROR_INFO_TO_EXOSE[prop]; + return !!ERROR_INFO_TO_EXPOSE[prop]; } function errorMessage(err) { From bb599025354ebccc978af7bed78fce13742cdcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:52:09 +0200 Subject: [PATCH 039/128] Refactoring tests hooks --- test/acceptance/user-database-timeout-limit.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index da9183c7..35add015 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -58,20 +58,18 @@ const createMapConfig = ({ describe('user database timeout limit', function () { describe('dataview', function () { beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); - }); - - beforeEach(function () { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - }); - - afterEach(function (done) { - this.testClient.drain(done); + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works but dataview request fails due to statement timeout', function (done) { From 5974413d5cead17efc5cdca3d6d0b77e4500d1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 19:08:29 +0200 Subject: [PATCH 040/128] Use 429 to indicate timeout errors --- lib/cartodb/controllers/base.js | 7 +++++++ test/acceptance/user-database-timeout-limit.js | 10 +++++----- test/acceptance/user-render-timeout-limit.js | 8 ++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index b8b73569..c5afdecb 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -301,5 +301,12 @@ function statusFromErrorMessage(errMsg) { statusCode = 404; } } + else if ( -1 !== errMsg.indexOf('Render timed out') ) { + statusCode = 429; // under discussion + } + else if ( -1 !== errMsg.indexOf('canceling statement due to statement timeout') ) { + statusCode = 429; // under discussion + } + return statusCode; } diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 35add015..53b69775 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -75,7 +75,7 @@ describe('user database timeout limit', function () { it('layergroup creation works but dataview request fails due to statement timeout', function (done) { const params = { response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -117,7 +117,7 @@ describe('user database timeout limit', function () { it('fails due to statement timeout', function (done) { const expectedResponse = { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -182,7 +182,7 @@ describe('user database timeout limit', function () { format: 'torque.json', layers: [ 0 ], response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -231,7 +231,7 @@ describe('user database timeout limit', function () { it('layergroup creation fails due to statement timeout', function (done) { const expectedResponse = { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -304,7 +304,7 @@ describe('user database timeout limit', function () { featureId: 1, layer: 0, response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index cd5b03a7..e579a132 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -85,7 +85,7 @@ describe('user render timeout limit', function () { it('layergroup creation fails due to statement timeout', function (done) { const expectedResponse = { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -174,7 +174,7 @@ describe('user render timeout limit', function () { it('layergroup creation works and render tile fails', function (done) { var params = { response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -211,7 +211,7 @@ describe('user render timeout limit', function () { const params = { format: 'mvt', response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -252,7 +252,7 @@ describe('user render timeout limit', function () { layers: 'mapnik', format: 'grid.json', response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/x-protobuf' } From 1fdc0621e780348c902e01a620eab3b24ed06a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 19:36:07 +0200 Subject: [PATCH 041/128] Categorize timeout errors --- lib/cartodb/controllers/base.js | 34 +++++++++++++++++++ .../acceptance/user-database-timeout-limit.js | 18 +++++++--- test/acceptance/user-render-timeout-limit.js | 27 ++++++++++++--- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index c5afdecb..cdfc84ff 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -195,6 +195,9 @@ BaseController.prototype.send = function(req, res, body, status, headers) { BaseController.prototype.sendError = function(req, res, err, label) { var allErrors = Array.isArray(err) ? err : [err]; + + allErrors = populateTimeoutErrors(allErrors) + label = label || 'UNKNOWN'; err = allErrors[0] || new Error(label); allErrors[0] = err; @@ -310,3 +313,34 @@ function statusFromErrorMessage(errMsg) { return statusCode; } + +function isRenderTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('Render timed out')) +} + +function isDatasourceTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')) +} + +function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); +} + +function populateTimeoutErrors (errors) { + return errors.map(function (error) { + if (isTimeoutError(error)) { + error.type = 'limit' + } + + if (isRenderTimeoutError(error)) { + error.subtype = 'render' + } + + if (isDatasourceTimeoutError(error)) { + + error.subtype = 'datasource' + } + + return error; + }) +} diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 53b69775..241b900c 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -87,7 +87,11 @@ describe('user database timeout limit', function () { assert.deepEqual(dataview, { errors: ['canceling statement due to statement timeout'], - errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'canceling statement due to statement timeout' + }] }); done(); @@ -127,7 +131,8 @@ describe('user database timeout limit', function () { assert.deepEqual(timeoutError, { errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], errors_with_context: [{ - type: 'layer', + type: 'limit', + subtype: 'datasource', message: 'TorqueRenderer: canceling statement due to statement timeout', layer: { id: 'torque-layer0', index: 0, type: 'torque' } }] @@ -195,7 +200,8 @@ describe('user database timeout limit', function () { assert.deepEqual(attributes, { errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], errors_with_context: [{ - type: 'unknown', + type: 'limit', + subtype: 'datasource', message: 'TorqueRenderer: canceling statement due to statement timeout', }] }); @@ -241,7 +247,8 @@ describe('user database timeout limit', function () { assert.deepEqual(timeoutError, { errors: [ 'canceling statement due to statement timeout' ], errors_with_context: [{ - type: 'layer', + type: 'limit', + subtype: 'datasource', message: 'canceling statement due to statement timeout', layer: { id: 'layer0', @@ -317,7 +324,8 @@ describe('user database timeout limit', function () { assert.deepEqual(attributes, { errors: ['canceling statement due to statement timeout'], errors_with_context: [{ - type: 'unknown', + type: 'limit', + subtype: 'datasource', message: 'canceling statement due to statement timeout' }] }); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index e579a132..c5751bed 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -97,7 +97,8 @@ describe('user render timeout limit', function () { assert.deepEqual(timeoutError, { errors: ["Render timed out"], errors_with_context: [{ - type: "layer", + type: 'limit', + subtype: 'render', message: "Render timed out", layer: { id: "layer0", @@ -181,10 +182,18 @@ describe('user render timeout limit', function () { } }; - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { assert.ifError(err); - assert.equal(tile.errors[0], 'Render timed out'); + assert.deepEqual(timeoutError, { + errors: ["Render timed out"], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: "Render timed out" + }] + }); + done(); }); }); @@ -223,7 +232,11 @@ describe('user render timeout limit', function () { assert.deepEqual(tile, { errors: ['Render timed out'], - errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: 'Render timed out' + }] }); done(); @@ -264,7 +277,11 @@ describe('user render timeout limit', function () { assert.deepEqual(tile, { errors: ['Render timed out'], - errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: 'Render timed out' + }] }); done(); From 44fc34b1ce5c4449fb11266dd2292f10e5cabfb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 20:07:31 +0200 Subject: [PATCH 042/128] Improve timeout error message --- lib/cartodb/controllers/base.js | 27 ++++++++----------- test/acceptance/limits.js | 12 ++++++--- .../acceptance/user-database-timeout-limit.js | 20 +++++++------- test/acceptance/user-render-timeout-limit.js | 16 +++++------ 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index cdfc84ff..5072d899 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -196,7 +196,7 @@ BaseController.prototype.send = function(req, res, body, status, headers) { BaseController.prototype.sendError = function(req, res, err, label) { var allErrors = Array.isArray(err) ? err : [err]; - allErrors = populateTimeoutErrors(allErrors) + allErrors = populateTimeoutErrors(allErrors); label = label || 'UNKNOWN'; err = allErrors[0] || new Error(label); @@ -304,22 +304,16 @@ function statusFromErrorMessage(errMsg) { statusCode = 404; } } - else if ( -1 !== errMsg.indexOf('Render timed out') ) { - statusCode = 429; // under discussion - } - else if ( -1 !== errMsg.indexOf('canceling statement due to statement timeout') ) { - statusCode = 429; // under discussion - } return statusCode; } function isRenderTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('Render timed out')) + return err.message && (-1 !== err.message.indexOf('Render timed out')); } function isDatasourceTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')) + return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); } function isTimeoutError (err) { @@ -328,19 +322,20 @@ function isTimeoutError (err) { function populateTimeoutErrors (errors) { return errors.map(function (error) { - if (isTimeoutError(error)) { - error.type = 'limit' - } - if (isRenderTimeoutError(error)) { - error.subtype = 'render' + error.subtype = 'render'; } if (isDatasourceTimeoutError(error)) { + error.subtype = 'datasource'; + } - error.subtype = 'datasource' + if (isTimeoutError(error)) { + error.message = 'You are over platform limits. Please contact us to know more details'; + error.type = 'limit'; + error.http_status = 429; } return error; - }) + }); } diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js index d0126623..f4156363 100644 --- a/test/acceptance/limits.js +++ b/test/acceptance/limits.js @@ -102,11 +102,13 @@ describe('render limits', function() { assert.response(server, createRequest(layergroup, user), { - status: 400 + status: 429 }, function(res) { var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, [ 'Render timed out' ]); + assert.deepEqual(parsed.errors, [ + 'You are over platform limits. Please contact us to know more details' + ]); done(); } ); @@ -167,11 +169,13 @@ describe('render limits', function() { encoding: 'binary' }, { - status: 400 + status: 429 }, function(res) { var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, ['Render timed out']); + assert.deepEqual(parsed.errors, [ + 'You are over platform limits. Please contact us to know more details' + ]); done(); } ); diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 241b900c..8662636f 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -86,11 +86,11 @@ describe('user database timeout limit', function () { assert.ifError(err); assert.deepEqual(dataview, { - errors: ['canceling statement due to statement timeout'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'canceling statement due to statement timeout' + message: 'You are over platform limits. Please contact us to know more details' }] }); @@ -129,11 +129,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors: [ 'You are over platform limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'TorqueRenderer: canceling statement due to statement timeout', + message: 'You are over platform limits. Please contact us to know more details', layer: { id: 'torque-layer0', index: 0, type: 'torque' } }] }); @@ -198,11 +198,11 @@ describe('user database timeout limit', function () { assert.ifError(err); assert.deepEqual(attributes, { - errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors: [ 'You are over platform limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'TorqueRenderer: canceling statement due to statement timeout', + message: 'You are over platform limits. Please contact us to know more details', }] }); @@ -245,11 +245,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'canceling statement due to statement timeout' ], + errors: [ 'You are over platform limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'canceling statement due to statement timeout', + message: 'You are over platform limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, @@ -322,11 +322,11 @@ describe('user database timeout limit', function () { assert.ifError(err); assert.deepEqual(attributes, { - errors: ['canceling statement due to statement timeout'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'canceling statement due to statement timeout' + message: 'You are over platform limits. Please contact us to know more details' }] }); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index c5751bed..a986dd34 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -95,11 +95,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["Render timed out"], + errors: ["You are over platform limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "Render timed out", + message: "You are over platform limits. Please contact us to know more details", layer: { id: "layer0", index: 0, @@ -186,11 +186,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["Render timed out"], + errors: ["You are over platform limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "Render timed out" + message: "You are over platform limits. Please contact us to know more details" }] }); @@ -231,11 +231,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['Render timed out'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'Render timed out' + message: 'You are over platform limits. Please contact us to know more details' }] }); @@ -276,11 +276,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['Render timed out'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'Render timed out' + message: 'You are over platform limits. Please contact us to know more details' }] }); From dd5209b9a724902142db0154502af62f320a7a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 09:39:37 +0200 Subject: [PATCH 043/128] Add torque.png timeout error test --- .../acceptance/user-database-timeout-limit.js | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 8662636f..6eb2f0e5 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -55,6 +55,15 @@ const createMapConfig = ({ } }); +const DATASOURCE_TIMEOUT_ERROR = { + errors: ['You are over platform limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details' + }] +}; + describe('user database timeout limit', function () { describe('dataview', function () { beforeEach(function (done) { @@ -85,21 +94,14 @@ describe('user database timeout limit', function () { this.testClient.getDataview('count', params, (err, dataview) => { assert.ifError(err); - assert.deepEqual(dataview, { - errors: ['You are over platform limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details' - }] - }); + assert.deepEqual(dataview, DATASOURCE_TIMEOUT_ERROR); done(); }); }); }); - describe('torque:', function () { + describe('torque', function () { describe('while validating in layergroup creation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ @@ -143,8 +145,8 @@ describe('user database timeout limit', function () { }); }); - describe('fetching "torque.json" tile', function () { - before(function (done) { + describe('fetching torque tiles', function () { + beforeEach(function (done) { const mapconfig = createMapConfig({ type: 'torque', cartocss: TestClient.CARTOCSS.TORQUE @@ -181,7 +183,7 @@ describe('user database timeout limit', function () { TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); }); - it('fails due to statement timeout', function (done) { + it('"torque.json" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'torque.json', @@ -197,14 +199,29 @@ describe('user database timeout limit', function () { this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { assert.ifError(err); - assert.deepEqual(attributes, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', - }] - }); + assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + + it('".png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'torque.png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { + assert.ifError(err); + + assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); done(); }); @@ -321,14 +338,7 @@ describe('user database timeout limit', function () { this.testClient.getAttributes(params, (err, res, attributes) => { assert.ifError(err); - assert.deepEqual(attributes, { - errors: ['You are over platform limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details' - }] - }); + assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); done(); }); From 0c4e67d6a8ea9f4aebdd9898cf5114e45064ecf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 10:21:39 +0200 Subject: [PATCH 044/128] Implemented database timeout test while requesting tiles --- .../acceptance/user-database-timeout-limit.js | 328 +++++++++++++++++- 1 file changed, 321 insertions(+), 7 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 6eb2f0e5..62aae5ce 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -11,12 +11,21 @@ const pointSleepSql = ` 2 val `; +const validationPointSleepSql = ` + SELECT + pg_sleep(0.3), + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + const createMapConfig = ({ version = '1.6.0', type = 'cartodb', sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', + interactivity = 'cartodb_id', countBy = 'cartodb_id', attributes } = {}) => ({ @@ -29,7 +38,8 @@ const createMapConfig = ({ }, cartocss, cartocss_version, - attributes + attributes, + interactivity } }], analyses: [ @@ -91,16 +101,320 @@ describe('user database timeout limit', function () { } }; - this.testClient.getDataview('count', params, (err, dataview) => { + this.testClient.getDataview('count', params, (err, timeoutError) => { assert.ifError(err); - assert.deepEqual(dataview, DATASOURCE_TIMEOUT_ERROR); + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); done(); }); }); }); + describe('raster', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching raster tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + + describe('vector', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching vector tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('"mvt" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'mvt', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + + + describe('interactivity', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching interactivity tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ interactivity: 'val' }); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it.skip('"grid.json" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'grid.json', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + describe('torque', function () { describe('while validating in layergroup creation', function () { beforeEach(function (done) { @@ -196,10 +510,10 @@ describe('user database timeout limit', function () { } }; - this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { assert.ifError(err); - assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); done(); }); @@ -335,10 +649,10 @@ describe('user database timeout limit', function () { } }; - this.testClient.getAttributes(params, (err, res, attributes) => { + this.testClient.getAttributes(params, (err, res, timeoutError) => { assert.ifError(err); - assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); done(); }); From 9af372381c0cdc3ce0693f7c99724eb17ab205a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 10:29:29 +0200 Subject: [PATCH 045/128] Fix content-type assertion --- test/acceptance/user-render-timeout-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index a986dd34..382a8035 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -267,7 +267,7 @@ describe('user render timeout limit', function () { response: { status: 429, headers: { - 'Content-Type': 'application/x-protobuf' + 'Content-Type': 'application/json; charset=utf-8' } } }; From aa4bb62f38153bdd0f91c423a557e49951a2c430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 10:29:46 +0200 Subject: [PATCH 046/128] Fix test --- test/acceptance/user-database-timeout-limit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 62aae5ce..1caeafec 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -390,11 +390,11 @@ describe('user database timeout limit', function () { TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); }); - it.skip('"grid.json" fails due to statement timeout', function (done) { + it('"grid.json" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'grid.json', - layers: [ 0 ], + layers: 'mapnik', response: { status: 429, headers: { From 91b3e373b756856785cb4516881218de41b6148c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 11:46:48 +0200 Subject: [PATCH 047/128] Add helper method to fetch static images --- test/support/test-client.js | 104 ++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 750a9f69..6eeb260d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -710,6 +710,110 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { ); }; +TestClient.prototype.getStaticCenter = function (params, callback) { + var self = this; + + let { layergroupid, z, lat, lng, width, height, format } = params + + var url = `/api/v1/map/`; + + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + step( + function createLayergroup() { + var next = this; + + if (layergroupid) { + return next(null, layergroupid); + } + + var data = self.mapConfig + var path = url; + + assert.response(self.server, + { + url: path, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(data) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).layergroupid); + } + ); + }, + function getStaticResult(err, _layergroupid) { + assert.ifError(err); + + var next = this; + + layergroupid = _layergroupid; + + url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}` + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + encoding: 'binary', + method: 'GET', + headers: { + host: 'localhost' + } + }; + + var expectedResponse = Object.assign({}, { + status: 200, + headers: { + 'Content-Type': 'image/png' + } + }, params.response); + + assert.response(self.server, request, expectedResponse, function(res, err) { + assert.ifError(err); + + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; + } + + next(null, res, body); + }); + }, + function finish(err, res, image) { + if (layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + return callback(err, res, image); + } + ); +}; + TestClient.prototype.getNodeStatus = function(nodeName, callback) { var self = this; From 40af73d524da922bef3c5b34d8fa86d2df8ab30a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 11:47:50 +0200 Subject: [PATCH 048/128] Implement test to check render timeout error for static api --- test/acceptance/user-render-timeout-limit.js | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 382a8035..23b3cc70 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -288,5 +288,105 @@ describe('user render timeout limit', function () { }); }); }); + + describe('static images', function () { + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but static image fails due to render timeout', function (done) { + const params = { + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png' + }; + + this.testClient.getStaticCenter(params, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + }); + + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works and render tile fails', function (done) { + const params = { + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, function (err, res, timeoutError) { + assert.ifError(err); + + assert.deepEqual(timeoutError, { + errors: ["You are over platform limits. Please contact us to know more details"], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: "You are over platform limits. Please contact us to know more details" + }] + }); + + done(); + }); + }); + }); + }); }); From 7d136031638cd93c2ede04fff753f0d3fa743d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 11:58:43 +0200 Subject: [PATCH 049/128] Implement test to validate database timeout error for static api --- .../acceptance/user-database-timeout-limit.js | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 1caeafec..a320d08f 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -208,6 +208,32 @@ describe('user database timeout limit', function () { done(); }); }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, function (err, res, timeoutError) { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); }); }); }); From 4c3e3005aa3cc51117577b7846990db3600f7317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:52:34 +0200 Subject: [PATCH 050/128] Apply asset fallback to database timeout errors --- lib/cartodb/server.js | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 13d64804..7968dc5f 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -118,8 +118,27 @@ module.exports = function(serverOptions) { var onTileErrorStrategy; if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { - if (err && err.message === 'Render timed out' && format === 'png') { - return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {}); + + function isRenderTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('Render timed out')); + } + + function isDatasourceTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); + } + + function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); + } + + function isRasterFormat (format) { + return format === 'png' || format === 'jpg' + } + + if (isTimeoutError(err) && isRasterFormat(format)) { + return callback(null, timeoutErrorTile, { + 'Content-Type': 'image/png', + }, {}); } else { return callback(err, tile, headers, stats); } From 05ddf1d505b4ccee25bb3589be5fc84d9154ad96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:53:29 +0200 Subject: [PATCH 051/128] Add test to check if asset fallback is working when enabled and database timeout erro happens --- .../acceptance/user-database-timeout-limit.js | 193 +++++++++++++----- 1 file changed, 138 insertions(+), 55 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index a320d08f..4b9df8a4 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -3,6 +3,8 @@ require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; + const pointSleepSql = ` SELECT pg_sleep(0.3), @@ -153,31 +155,6 @@ describe('user database timeout limit', function () { }); describe('fetching raster tiles', function () { - beforeEach(function (done) { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - const expectedResponse = { - status: 200, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - this.testClient.getLayergroup(expectedResponse, (err, res) => { - if (err) { - return done(err); - } - - this.layergroupid = res.layergroupid; - - done(); - }); - }); - - afterEach(function (done) { - this.testClient.drain(done); - }); - describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); @@ -187,51 +164,157 @@ describe('user database timeout limit', function () { TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); }); - it('"png" fails due to statement timeout', function (done) { - const params = { - layergroupid: this.layergroupid, - format: 'png', - layers: [ 0 ], - response: { - status: 429, + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, headers: { 'Content-Type': 'application/json; charset=utf-8' } - } - }; + }; - this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { - assert.ifError(err); + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + this.layergroupid = res.layergroupid; - done(); + done(); + }); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.drain(done); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ] + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png' + }; + + this.testClient.getStaticCenter(params, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); }); }); - it('"static png" fails due to statement timeout', function (done) { - const params = { - layergroupid: this.layergroupid, - zoom: 0, - lat: 0, - lng: 0, - width: 256, - height: 256, - format: 'png', - response: { - status: 429, + describe('with onTileErrorStrategy DISABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, headers: { 'Content-Type': 'application/json; charset=utf-8' } - } - }; + }; - this.testClient.getStaticCenter(params, function (err, res, timeoutError) { - assert.ifError(err); + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + this.layergroupid = res.layergroupid; - done(); + done(); + }); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.drain(done); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); }); }); }); From 46fee774bdba8b9c54323e3e0d17e530c0dcaf56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:54:41 +0200 Subject: [PATCH 052/128] Fix misconfiguration in test's hook --- test/acceptance/user-render-timeout-limit.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 23b3cc70..2fab657f 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -303,6 +303,8 @@ describe('user render timeout limit', function () { }); afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { if (err) { return done(err); From 9258ad7eccb8663fbd58885e96536929b4de8258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:56:03 +0200 Subject: [PATCH 053/128] Fix style typo --- lib/cartodb/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7968dc5f..7b6fc33a 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -132,7 +132,7 @@ module.exports = function(serverOptions) { } function isRasterFormat (format) { - return format === 'png' || format === 'jpg' + return format === 'png' || format === 'jpg'; } if (isTimeoutError(err) && isRasterFormat(format)) { From 262f854e68dfe787525a5962ddc49a557fd6bb21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 13:10:55 +0200 Subject: [PATCH 054/128] Remove error wrapping --- lib/cartodb/models/dataview/base.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 7c3dbe6f..90d8a299 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -10,8 +10,7 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { this.sql(psql, override, function(err, query) { psql.query(query, function(err, result) { if (err) { - var error = new Error(err.message); - return callback(error, result); + return callback(err, result); } result = self.format(result, override); From 0594407b3837b651944c69ebf5b703f8092363e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 15:03:09 +0200 Subject: [PATCH 055/128] Change error message --- lib/cartodb/controllers/base.js | 2 +- test/acceptance/limits.js | 4 ++-- .../acceptance/user-database-timeout-limit.js | 24 +++++++++---------- test/acceptance/user-render-timeout-limit.js | 20 ++++++++-------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 5072d899..e3d2a407 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -331,7 +331,7 @@ function populateTimeoutErrors (errors) { } if (isTimeoutError(error)) { - error.message = 'You are over platform limits. Please contact us to know more details'; + error.message = 'You are over platform\'s limits. Please contact us to know more details'; error.type = 'limit'; error.http_status = 429; } diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js index f4156363..372fb189 100644 --- a/test/acceptance/limits.js +++ b/test/acceptance/limits.js @@ -107,7 +107,7 @@ describe('render limits', function() { function(res) { var parsed = JSON.parse(res.body); assert.deepEqual(parsed.errors, [ - 'You are over platform limits. Please contact us to know more details' + 'You are over platform\'s limits. Please contact us to know more details' ]); done(); } @@ -174,7 +174,7 @@ describe('render limits', function() { function(res) { var parsed = JSON.parse(res.body); assert.deepEqual(parsed.errors, [ - 'You are over platform limits. Please contact us to know more details' + 'You are over platform\'s limits. Please contact us to know more details' ]); done(); } diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 4b9df8a4..e1ff7bda 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -68,11 +68,11 @@ const createMapConfig = ({ }); const DATASOURCE_TIMEOUT_ERROR = { - errors: ['You are over platform limits. Please contact us to know more details'], + errors: ['You are over platform\'s limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details' + message: 'You are over platform\'s limits. Please contact us to know more details' }] }; @@ -140,11 +140,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -348,11 +348,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -450,11 +450,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -554,11 +554,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'torque-layer0', index: 0, type: 'torque' } }] }); @@ -685,11 +685,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 2fab657f..45e41f89 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -95,11 +95,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform limits. Please contact us to know more details"], + errors: ["You are over platform\'s limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform limits. Please contact us to know more details", + message: "You are over platform\'s limits. Please contact us to know more details", layer: { id: "layer0", index: 0, @@ -186,11 +186,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform limits. Please contact us to know more details"], + errors: ["You are over platform\'s limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform limits. Please contact us to know more details" + message: "You are over platform\'s limits. Please contact us to know more details" }] }); @@ -231,11 +231,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['You are over platform limits. Please contact us to know more details'], + errors: ['You are over platform\'s limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'You are over platform limits. Please contact us to know more details' + message: 'You are over platform\'s limits. Please contact us to know more details' }] }); @@ -276,11 +276,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['You are over platform limits. Please contact us to know more details'], + errors: ['You are over platform\'s limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'You are over platform limits. Please contact us to know more details' + message: 'You are over platform\'s limits. Please contact us to know more details' }] }); @@ -377,11 +377,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform limits. Please contact us to know more details"], + errors: ["You are over platform\'s limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform limits. Please contact us to know more details" + message: "You are over platform\'s limits. Please contact us to know more details" }] }); From ab879e263465e024af45b235952d2c2760a902b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 19:13:55 +0200 Subject: [PATCH 056/128] Use new version of `getUserTimeoutRenderLimits` --- lib/cartodb/api/user_limits_api.js | 17 ++++++++++------- yarn.lock | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 7e004e31..8954c261 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -87,13 +87,16 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call return next(err); } - self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, next); + self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { + if (err) { + return next(err) + } + + next(null, { + render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic + }); + }); }, - function setTilerRenderLimit(err, timeoutRenderLimit) { - if (err) { - return callback(err); - } - return callback(null, timeoutRenderLimit); - } + callback ); }; diff --git a/yarn.lock b/yarn.lock index d7c06ac0..63d9f6fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -260,7 +260,7 @@ cartodb-query-tables@0.2.0: cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: version "0.13.3" - resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/0c0e5c410243bd4a16e9d94b52821e9d6533eb16" + resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/530ae2736a184a3b9c9e978efa7dcb4faae7abee" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" From 3574700c2d4c13cecffc438105ecaa645833067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Aug 2017 11:07:44 +0200 Subject: [PATCH 057/128] Remove tiler render limit --- lib/cartodb/api/user_limits_api.js | 21 +- test/acceptance/limits.js | 315 ----------------------------- yarn.lock | 2 +- 3 files changed, 2 insertions(+), 336 deletions(-) delete mode 100644 test/acceptance/limits.js diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 8954c261..74d0cf21 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -24,21 +24,6 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) }; step( - function getTilerLimit() { - var next = this; - self.getTilerRenderLimit(username, function (err, tilerRenderLimit) { - if (err) { - return callback(err); - } - - if (Number.isFinite(tilerRenderLimit)) { - limits.render = tilerRenderLimit; - return callback(null, limits); - } - - return next(); - }); - }, function getTimeoutLimit() { self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { if (err) { @@ -57,10 +42,6 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) ); }; -UserLimitsApi.prototype.getTilerRenderLimit = function (username, callback) { - this.metadataBackend.getTilerRenderLimit(username, callback); -}; - UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { var self = this; @@ -89,7 +70,7 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { if (err) { - return next(err) + return next(err); } next(null, { diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js deleted file mode 100644 index 372fb189..00000000 --- a/test/acceptance/limits.js +++ /dev/null @@ -1,315 +0,0 @@ -var testHelper = require('../support/test_helper'); - -var assert = require('../support/assert'); -var _ = require('underscore'); -var redis = require('redis'); - -var CartodbWindshaft = require('../../lib/cartodb/server'); -var serverOptions = require('../../lib/cartodb/server_options'); - -var LayergroupToken = require('../support/layergroup-token'); - -describe('render limits', function() { - - var layergroupUrl = '/api/v1/map'; - - var redisClient = redis.createClient(global.environment.redis.port); - - var server; - var keysToDelete; - beforeEach(function() { - keysToDelete = {}; - server = new CartodbWindshaft(serverOptions); - server.setMaxListeners(0); - }); - - afterEach(function(done) { - testHelper.deleteRedisKeys(keysToDelete, done); - }); - - var user = 'localhost'; - - var pointSleepSql = "SELECT pg_sleep(0.5)," + - " 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id"; - var pointCartoCss = '#layer { marker-fill:red; }'; - var polygonSleepSql = "SELECT pg_sleep(0.5)," + - " ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id"; - var polygonCartoCss = '#layer { polygon-fill:red; }'; - - function singleLayergroupConfig(sql, cartocss) { - return { - version: '1.0.0', - layers: [ - { - type: 'mapnik', - options: { - sql: sql, - cartocss: cartocss, - cartocss_version: '2.0.1' - } - } - ] - }; - } - - function createRequest(layergroup, userHost) { - return { - url: layergroupUrl, - method: 'POST', - headers: { - host: userHost, - 'Content-Type': 'application/json' - }, - data: JSON.stringify(layergroup) - }; - } - - function withRenderLimit(user, renderLimit, callback) { - redisClient.SELECT(5, function(err) { - if (err) { - return callback(err); - } - var userLimitsKey = 'limits:tiler:' + user; - redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) { - if (err) { - return callback(err); - } - keysToDelete[userLimitsKey] = 5; - return callback(); - }); - }); - - } - - describe('with onTileErrorStrategy DISABLED', function() { - var onTileErrorStrategyEnabled; - before(function() { - onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy; - global.environment.enabledFeatures.onTileErrorStrategy = false; - }); - - after(function() { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled; - }); - - it("layergroup creation fails if test tile is slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 429 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, [ - 'You are over platform\'s limits. Please contact us to know more details' - ]); - done(); - } - ); - }); - }); - - it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) { - withRenderLimit(user, 5000, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.ok(parsed.layergroupid); - keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - done(); - } - ); - }); - }); - - - it("layergroup creation works if test tile is fast but tile request fails if they are slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 429 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, [ - 'You are over platform\'s limits. Please contact us to know more details' - ]); - done(); - } - ); - - } - ); - }); - }); - - it("tile request does not fail if user limit is high enough", function(done) { - withRenderLimit(user, 5000, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 200, - headers: { - 'Content-Type': 'image/png' - } - }, - function(res, err) { - done(err); - } - ); - - } - ); - }); - }); - - }); - - describe('with onTileErrorStrategy', function() { - - it("layergroup creation works even if test tile is slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.ok(parsed.layergroupid); - keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - done(); - } - ); - }); - }); - - it("layergroup creation and tile requests works even if they are slow but returns fallback", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 200, - headers: { - 'Content-Type': 'image/png' - } - }, - function(res, err) { - if (err) { - done(err); - } - var referenceImagePath = './test/fixtures/render-timeout-fallback.png'; - assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25, - function(imgErr/*, similarity*/) { - done(imgErr); - } - ); - } - ); - - } - ); - }); - }); - - }); - -}); diff --git a/yarn.lock b/yarn.lock index 63d9f6fe..ebc2c0d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -260,7 +260,7 @@ cartodb-query-tables@0.2.0: cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: version "0.13.3" - resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/530ae2736a184a3b9c9e978efa7dcb4faae7abee" + resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/4c0586009dbfd464eb9e742bed39170334f40453" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" From 1bc5c044890cfe2ed169a93f7887275017cfa33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Aug 2017 13:15:40 +0200 Subject: [PATCH 058/128] Remove unused method --- test/support/test-client.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 6eeb260d..15448f68 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1097,7 +1097,4 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) ); }; -TestClient.cleanDatabaseConnections = function () { - helper.cleanPGPoolConnections() -} From 660078f284caf65af724209f7c80b8b5dc48bfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 16:53:08 +0200 Subject: [PATCH 059/128] Fix minor issues with timezones --- NEWS.md | 6 + lib/cartodb/models/dataview/histogram.js | 4 +- test/acceptance/dataviews/histogram.js | 185 ++++++++++++++++++++++- 3 files changed, 192 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9ca6d4a4..3d5d5722 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,12 @@ ## 3.10.2 Released 2017-mm-dd +Announcements: + - Allow to override with any aggregation for histograms instantiated w/o aggregation. + +Bug fixes: + - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + ## 3.10.1 Released 2017-08-04 diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 76b42c03..891cc2ec 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -246,7 +246,7 @@ var dateHistogramQueryTpl = dot.template([ ' date_part(', ' \'epoch\', ', ' date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', @@ -333,7 +333,7 @@ Histogram.prototype._buildQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; - if (this._columnType === 'date' && this.aggregation !== undefined) { + if (this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined)) { return this._buildDateHistogramQuery(psql, override, callback); } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index a75cf2b1..84e2c15e 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -888,7 +888,6 @@ describe('histogram-dates: aggregation input value', function() { }); }); - describe('histogram-dates: timestamp starts at epoch', function() { afterEach(function(done) { @@ -958,3 +957,187 @@ describe('histogram-dates: timestamp starts at epoch', function() { }); }); }); + +describe('histogram-dates: trunc timestamp for each bin respecting user\'s timezone', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should return histogram with two buckets', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); +}); + + +describe('histogram: be able to override with aggregation for histograms instantiated w/o aggregation', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should apply aggregation to the histogram', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); +}); From 1f8da14c2ae6dff1e3353d04ac821f89cfe36430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 18:27:24 +0200 Subject: [PATCH 060/128] Cast to timestamp start_date and end_date to calculate bins when date column is timestamptz --- lib/cartodb/models/dataview/histogram.js | 4 +- test/acceptance/dataviews/histogram.js | 88 ++++++++++++++++-------- 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 891cc2ec..a044bfc7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -180,9 +180,9 @@ var dateBasicsQueryTpl = dot.template([ ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._offset}}\'', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )) AS start_date,', - ' max({{=it._column}} AT TIME ZONE \'{{=it._offset}}\') AS end_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 84e2c15e..3baf002b 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -991,6 +991,16 @@ describe('histogram-dates: trunc timestamp for each bin respecting user\'s timez column: 'd', aggregation: 'auto' } + }, + timezone_epoch_histogram_tz: { + source: { + id: 'a1' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } } }, [ @@ -1007,45 +1017,69 @@ describe('histogram-dates: trunc timestamp for each bin respecting user\'s timez ') date' ].join(' ') } + }, + { + id: 'a1', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamptz,', + '\'1970-01-01 01:59:00\'::timestamptz,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } } ] ); - it('should return histogram with two buckets', function(done) { - this.testClient = new TestClient(mapConfig, 1234); + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'timezone_epoch_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'timezone_epoch_histogram' + }]; - const override = { - aggregation: 'day', - offset: '-3600' - }; + dateHistogramsUseCases.forEach(function (test) { + it('should return histogram with two buckets ' + test.desc , function(done) { + this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { - assert.ifError(err); + const override = { + aggregation: 'day', + offset: '-3600' + }; - var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 - var initialTimestamp = '1969-12-31T00:00:00-01:00'; - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(OFFSET_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); + this.testClient.getDataview(test.dataviewId, override, function(err, dataview) { + assert.ifError(err); - dataview.bins.forEach(function (bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(OFFSET_IN_MINUTES) - .add(index, override.aggregation) - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(OFFSET_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); }); }); From 2024e89c6ab227091e680d5cf74c480ccaf1795d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 18:30:17 +0200 Subject: [PATCH 061/128] Update news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 3d5d5722..ced38066 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,7 @@ Announcements: Bug fixes: - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + - Support timestamp with timezones to calculate the number of bins in time-series. ## 3.10.1 From b2bbc329ea9a542925644f5e3a6b540ffb6e2fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:03:49 +0200 Subject: [PATCH 062/128] Apply prefix for intermediate query variables to avoid name colision --- lib/cartodb/models/dataview/histogram.js | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index a044bfc7..6d399d39 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -176,14 +176,14 @@ var histogramQueryTpl = dot.template([ var dateBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', - ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', - ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(date_trunc(', ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )) AS start_date,', - ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS end_date,', - ' count(1) AS total_rows', + ' )) AS __cdb_start_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' ].join(' \n')); @@ -191,19 +191,19 @@ var dateBasicsQueryTpl = dot.template([ var dateOverrideBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val,', - ' min({{=it._start}}) AS min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' max({{=it._end}}) AS __cdb_max_val,', + ' min({{=it._start}}) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(', ' date_trunc(', ' \'{{=it._aggregation}}\',', ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )', - ' ) AS start_date,', + ' ) AS __cdb_start_date,', ' max(', ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AS end_date,', - ' count(1) AS total_rows', + ' ) AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' ].join(' \n')); @@ -217,8 +217,8 @@ var dateBinsQueryTpl = dot.template([ ' SELECT', ' ARRAY(', ' SELECT GENERATE_SERIES(', - ' start_date::timestamptz,', - ' end_date::timestamptz,', + ' __cdb_start_date::timestamptz,', + ' __cdb_end_date::timestamptz,', ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', ' ) AS bins_array', @@ -229,10 +229,10 @@ var dateBinsQueryTpl = dot.template([ var dateHistogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' (__cdb_max_val - __cdb_min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', - ' CASE WHEN min_val = max_val', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', @@ -250,14 +250,14 @@ var dateHistogramQueryTpl = dot.template([ ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', - ' date_part(\'epoch\', start_date)::numeric AS timestamp_start,', + ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, __cdb_start_date', 'ORDER BY bin' ].join('\n')); From 0f45675652a87488ba261e0c1035fed650d6ed78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:06:31 +0200 Subject: [PATCH 063/128] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index ced38066..bd89d748 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ Announcements: Bug fixes: - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. - Support timestamp with timezones to calculate the number of bins in time-series. + - Fixed issue related to name collision while building time-series query. ## 3.10.1 From 5c80ff81917f4093a1ac9c3e3dc3964b4b17337a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:24:15 +0200 Subject: [PATCH 064/128] Extract query: move condition to a method --- lib/cartodb/models/dataview/histogram.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6d399d39..4af16d65 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -328,12 +328,16 @@ Histogram.prototype.sql = function(psql, override, callback) { this._buildQuery(psql, override, callback); }; +Histogram.prototype.isDateHistogram = function (override) { + return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); +}; + Histogram.prototype._buildQuery = function (psql, override, callback) { var filteredQuery, basicsQuery, binsQuery; var _column = this.column; var _query = this.query; - if (this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined)) { + if (this.isDateHistogram(override)) { return this._buildDateHistogramQuery(psql, override, callback); } From 994e58bef7e2f9f63a66b6c04e4bf9d38428915f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:33:26 +0200 Subject: [PATCH 065/128] Add prefix to bins_number and nulls_cout to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4af16d65..dce22bb0 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -109,7 +109,7 @@ var binsQueryTpl = dot.template([ ' {{=it._maxBins}}', ' )', ' )', - ' END AS bins_number', + ' END AS __cdb_bins_number', ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' @@ -117,14 +117,14 @@ var binsQueryTpl = dot.template([ var overrideBinsQueryTpl = dot.template([ 'bins AS (', - ' SELECT {{=it._bins}} AS bins_number', + ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); var nullsQueryTpl = dot.template([ 'nulls AS (', ' SELECT', - ' count(*) AS nulls_count', + ' count(*) AS __cdb_nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' @@ -153,22 +153,22 @@ var nansQueryTpl = dot.template([ var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', + ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', ' {{?it._isFloatColumn}}infinities_count,', ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number), __cdb_bins_number)) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', - 'GROUP BY bin, bins_number, bin_width, nulls_count,', + 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -212,7 +212,7 @@ var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', ' bins_array,', - ' ARRAY_LENGTH(bins_array, 1) AS bins_number', + ' ARRAY_LENGTH(bins_array, 1) AS __cdb_bins_number', ' FROM (', ' SELECT', ' ARRAY(', @@ -229,9 +229,9 @@ var dateBinsQueryTpl = dot.template([ var dateHistogramQueryTpl = dot.template([ 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', @@ -239,7 +239,7 @@ var dateHistogramQueryTpl = dot.template([ ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', ' bins_array', ' ),', - ' bins_number', + ' __cdb_bins_number', ' )) - 1', ' END AS bin,', ' min(', @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, __cdb_start_date', + 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count, __cdb_avg_val, __cdb_start_date', 'ORDER BY bin' ].join('\n')); From 823925d09143b75e96c1f9d308bba741ce7ecf4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:41:22 +0200 Subject: [PATCH 066/128] Add prefix to bins_array to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dce22bb0..c6f4ffbb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -211,8 +211,8 @@ var dateOverrideBasicsQueryTpl = dot.template([ var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', - ' bins_array,', - ' ARRAY_LENGTH(bins_array, 1) AS __cdb_bins_number', + ' __cdb_bins_array,', + ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', ' FROM (', ' SELECT', ' ARRAY(', @@ -221,9 +221,9 @@ var dateBinsQueryTpl = dot.template([ ' __cdb_end_date::timestamptz,', ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', - ' ) AS bins_array', + ' ) AS __cdb_bins_array', ' FROM basics', - ' ) _cdb_bins_array', + ' ) _cdb_bins_array_query', ')' ].join('\n')); @@ -237,7 +237,7 @@ var dateHistogramQueryTpl = dot.template([ ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', - ' bins_array', + ' __cdb_bins_array', ' ),', ' __cdb_bins_number', ' )) - 1', From e4461601512b40f599e62e97221d6c8690d334ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:45:05 +0200 Subject: [PATCH 067/128] Use final columns to group by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c6f4ffbb..d868513a 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count, __cdb_avg_val, __cdb_start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, timestamp_start', 'ORDER BY bin' ].join('\n')); From 91d5a0e4e4c38540606ac5bf322c02a2fe360c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:46:26 +0200 Subject: [PATCH 068/128] Remove column avg_val in group_by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index d868513a..46f86a5d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, timestamp_start', + 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' ].join('\n')); From 0f8de9e74b920d0d8ab20ce379bf930c44011756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:06:36 +0200 Subject: [PATCH 069/128] Add prefix to basics query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 28 ++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 46f86a5d..c9204960 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -67,7 +67,7 @@ var filteredQueryTpl = dot.template([ ].join(' \n')); var basicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', @@ -76,7 +76,7 @@ var basicsQueryTpl = dot.template([ ].join(' \n')); var overrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', @@ -110,7 +110,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS __cdb_bins_number', - ' FROM basics, iqrange, filtered_source', + ' FROM __cdb_basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -161,20 +161,26 @@ var histogramQueryTpl = dot.template([ ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number), __cdb_bins_number)) - 1', + ' ELSE GREATEST(', + ' 1,', + ' LEAST(', + ' WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number),', + ' __cdb_bins_number', + ' )', + ' ) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); var dateBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', @@ -184,12 +190,12 @@ var dateBasicsQueryTpl = dot.template([ ' )) AS __cdb_start_date,', ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM ({{=it._query}}) __cdb_basics_query', ')' ].join(' \n')); var dateOverrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._end}}) AS __cdb_max_val,', ' min({{=it._start}}) AS __cdb_min_val,', @@ -204,7 +210,7 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AS __cdb_end_date,', ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM ({{=it._query}}) __cdb_basics_query', ')' ].join(' \n')); @@ -222,7 +228,7 @@ var dateBinsQueryTpl = dot.template([ ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', ' ) AS __cdb_bins_array', - ' FROM basics', + ' FROM __cdb_basics', ' ) _cdb_bins_array_query', ')' ].join('\n')); @@ -255,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From f3ababffc1f29e8f2fb8e5fed34b158888e56429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:12:52 +0200 Subject: [PATCH 070/128] Prefix bins query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c9204960..2af66449 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -99,7 +99,7 @@ var iqrQueryTpl = dot.template([ ].join('\n')); var binsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', ' THEN 1', ' ELSE GREATEST(', @@ -116,7 +116,7 @@ var binsQueryTpl = dot.template([ ].join('\n')); var overrideBinsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' @@ -215,7 +215,7 @@ var dateOverrideBasicsQueryTpl = dot.template([ ].join(' \n')); var dateBinsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT', ' __cdb_bins_array,', ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', @@ -261,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From 3ee064a59fa87cb2d0802568af1fd8a9d190d465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:32:13 +0200 Subject: [PATCH 071/128] Prefix nulls query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 2af66449..54a9fcf6 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -122,7 +122,7 @@ var overrideBinsQueryTpl = dot.template([ ].join('\n')); var nullsQueryTpl = dot.template([ - 'nulls AS (', + '__cdb_nulls AS (', ' SELECT', ' count(*) AS __cdb_nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' @@ -261,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From a9e3bc3cdadfbfd1d57cf609442096499f1100c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:36:23 +0200 Subject: [PATCH 072/128] Prefix filtered source to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 54a9fcf6..629a69f7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -52,9 +52,9 @@ var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; var filteredQueryTpl = dot.template([ - 'filtered_source AS (', + '__cdb_filtered_source AS (', ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source', + ' FROM ({{=it._query}}) _cdb_filtered_source_query', ' WHERE', ' {{=it._column}} IS NOT NULL', ' {{?it._isFloatColumn}}AND', @@ -71,7 +71,7 @@ var basicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' FROM __cdb_filtered_source', ')' ].join(' \n')); @@ -80,7 +80,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' FROM __cdb_filtered_source', ')' ].join('\n')); @@ -91,7 +91,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM filtered_source) _cdb_quartiles', + ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -110,7 +110,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS __cdb_bins_number', - ' FROM __cdb_basics, iqrange, filtered_source', + ' FROM __cdb_basics, iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From 7e058955ea71ac88010855c3d3d03fc56226e22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:39:51 +0200 Subject: [PATCH 073/128] Use final naming to group by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 629a69f7..7099340d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -174,7 +174,7 @@ var histogramQueryTpl = dot.template([ ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', - 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', + 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); From 34cf45bc9dedbfbc4595ac1e7a8151b606e90716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:43:31 +0200 Subject: [PATCH 074/128] Prefix infinities query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7099340d..0edfaae9 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -131,7 +131,7 @@ var nullsQueryTpl = dot.template([ ].join('\n')); var infinitiesQueryTpl = dot.template([ - 'infinities AS (', + '__cdb_infinities AS (', ' SELECT', ' count(*) AS infinities_count', ' FROM ({{=it._query}}) _cdb_histogram_infinities', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From c7e5dbf1585a541fe2b5328f576896770ecc691d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:46:34 +0200 Subject: [PATCH 075/128] Fix query aliases --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 0edfaae9..01335421 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -134,7 +134,7 @@ var infinitiesQueryTpl = dot.template([ '__cdb_infinities AS (', ' SELECT', ' count(*) AS infinities_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' FROM ({{=it._query}}) __cdb_infinities_query', ' WHERE', ' {{=it._column}} = \'infinity\'::float', ' OR', @@ -146,7 +146,7 @@ var nansQueryTpl = dot.template([ 'nans AS (', ' SELECT', ' count(*) AS nans_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' FROM ({{=it._query}}) __cdb_nans_query', ' WHERE {{=it._column}} = \'NaN\'::float', ')' ].join('\n')); From cf71489a7f3c18718e0aefdd5c5b84997bc26960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:54:49 +0200 Subject: [PATCH 076/128] Prefix nans and infinities counters to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 01335421..26ee452c 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -133,7 +133,7 @@ var nullsQueryTpl = dot.template([ var infinitiesQueryTpl = dot.template([ '__cdb_infinities AS (', ' SELECT', - ' count(*) AS infinities_count', + ' count(*) AS __cdb_infinities_count', ' FROM ({{=it._query}}) __cdb_infinities_query', ' WHERE', ' {{=it._column}} = \'infinity\'::float', @@ -143,9 +143,9 @@ var infinitiesQueryTpl = dot.template([ ].join('\n')); var nansQueryTpl = dot.template([ - 'nans AS (', + '__cdb_nans AS (', ' SELECT', - ' count(*) AS nans_count', + ' count(*) AS __cdb_nans_count', ' FROM ({{=it._query}}) __cdb_nans_query', ' WHERE {{=it._column}} = \'NaN\'::float', ')' @@ -156,8 +156,8 @@ var histogramQueryTpl = dot.template([ ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', ' __cdb_bins_number AS bins_number,', ' __cdb_nulls_count AS nulls_count,', - ' {{?it._isFloatColumn}}infinities_count,', - ' nans_count,{{?}}', + ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', + ' __cdb_nans_count AS nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From ff3d7ed7b21829194518a2d75a84c80d3d4bc126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:57:12 +0200 Subject: [PATCH 077/128] Fix jshint typo --- lib/cartodb/models/dataview/histogram.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 26ee452c..ec3e616e 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -173,7 +173,8 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', + ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From 7e159c565b61ffdb2bfdbe3e385b5d6daae8541d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:03:40 +0200 Subject: [PATCH 078/128] Prefix iqr query calculation to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index ec3e616e..9c63b836 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -85,8 +85,8 @@ var overrideBasicsQueryTpl = dot.template([ ].join('\n')); var iqrQueryTpl = dot.template([ - 'iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + '__cdb_iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', ' FROM (', ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', @@ -100,17 +100,17 @@ var iqrQueryTpl = dot.template([ var binsQueryTpl = dot.template([ '__cdb_bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + ' SELECT CASE WHEN total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((max_val - min_val) / (2 * __cdb_iqr * power(total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', ' END AS __cdb_bins_number', - ' FROM __cdb_basics, iqrange, __cdb_filtered_source', + ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); From 3c061769c68582eb645a8e6184bbd3cedd13046e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:11:35 +0200 Subject: [PATCH 079/128] Prefix basics columns to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 9c63b836..7d29d01d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -69,8 +69,8 @@ var filteredQueryTpl = dot.template([ var basicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', ' FROM __cdb_filtered_source', ')' ].join(' \n')); @@ -78,8 +78,8 @@ var basicsQueryTpl = dot.template([ var overrideBasicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', ' FROM __cdb_filtered_source', ')' ].join('\n')); @@ -100,12 +100,12 @@ var iqrQueryTpl = dot.template([ var binsQueryTpl = dot.template([ '__cdb_bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR __cdb_iqr = 0', + ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * __cdb_iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', @@ -153,18 +153,18 @@ var nansQueryTpl = dot.template([ var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', ' __cdb_bins_number AS bins_number,', ' __cdb_nulls_count AS nulls_count,', ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', ' __cdb_nans_count AS nans_count,{{?}}', - ' avg_val,', - ' CASE WHEN min_val = max_val', + ' __cdb_avg_val AS avg_val,', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(', ' 1,', ' LEAST(', - ' WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number),', + ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', ' __cdb_bins_number', ' )', ' ) - 1', From e7a0b246a39cd722138196c68b657b94bd1154e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:20:36 +0200 Subject: [PATCH 080/128] Prefix with double underscore --- lib/cartodb/models/dataview/histogram.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7d29d01d..b07aa474 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -54,7 +54,7 @@ var BIN_MAX_NUMBER = 48; var filteredQueryTpl = dot.template([ '__cdb_filtered_source AS (', ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source_query', + ' FROM ({{=it._query}}) __cdb_filtered_source_query', ' WHERE', ' {{=it._column}} IS NOT NULL', ' {{?it._isFloatColumn}}AND', @@ -94,7 +94,7 @@ var iqrQueryTpl = dot.template([ ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', - ' ) _cdb_iqr', + ' ) __cdb_iqr', ')' ].join('\n')); @@ -125,7 +125,7 @@ var nullsQueryTpl = dot.template([ '__cdb_nulls AS (', ' SELECT', ' count(*) AS __cdb_nulls_count', - ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' FROM ({{=it._query}}) __cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' ].join('\n')); @@ -230,7 +230,7 @@ var dateBinsQueryTpl = dot.template([ ' )', ' ) AS __cdb_bins_array', ' FROM __cdb_basics', - ' ) _cdb_bins_array_query', + ' ) __cdb_bins_array_query', ')' ].join('\n')); From c643160671b5a40dec233192d33a6aaafc0667e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:32:53 +0200 Subject: [PATCH 081/128] Prefix date interval query (to calculate automatic aggregation) to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 50 ++++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b07aa474..c27c696f 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -9,42 +9,42 @@ var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var dateIntervalQueryTpl = dot.template([ 'WITH', - 'dates AS (', + '__cdb_dates AS (', ' SELECT', - ' MAX({{=it.column}}::timestamp) AS _end,', - ' MIN({{=it.column}}::timestamp) AS _start', - ' FROM ({{=it.query}}) _cdb_source', + ' MAX({{=it.column}}::timestamp) AS __cdb_end,', + ' MIN({{=it.column}}::timestamp) AS __cdb_start', + ' FROM ({{=it.query}}) __cdb_source', '),', - 'interval_in_days AS (', + '__cdb_interval_in_days AS (', ' SELECT' , - ' DATE_PART(\'day\', _end - _start) AS days', - ' FROM dates', + ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', + ' FROM __cdb_dates', '),', - 'interval_in_hours AS (', + '__cdb_interval_in_hours AS (', ' SELECT', - ' days * 24 + DATE_PART(\'hour\', _end - _start) AS hours', - ' FROM interval_in_days, dates', + ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', + ' FROM __cdb_interval_in_days, __cdb_dates', '),', - 'interval_in_minutes AS (', + '__cdb_interval_in_minutes AS (', ' SELECT', - ' hours * 60 + DATE_PART(\'minute\', _end - _start) AS minutes', - ' FROM interval_in_hours, dates', + ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', + ' FROM __cdb_interval_in_hours, __cdb_dates', '),', - 'interval_in_seconds AS (', + '__cdb_interval_in_seconds AS (', ' SELECT', - ' minutes * 60 + DATE_PART(\'second\', _end - _start) AS seconds', - ' FROM interval_in_minutes, dates', + ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', + ' FROM __cdb_interval_in_minutes, __cdb_dates', ')', 'SELECT', - ' ROUND(days / 365) AS year,', - ' ROUND(days / 90) AS quarter,', - ' ROUND(days / 30) AS month,', - ' ROUND(days / 7) AS week,', - ' days AS day,', - ' hours AS hour,', - ' minutes AS minute,', - ' seconds AS second', - 'FROM interval_in_days, interval_in_hours, interval_in_minutes, interval_in_seconds' + ' ROUND(__cdb_days / 365) AS year,', + ' ROUND(__cdb_days / 90) AS quarter,', + ' ROUND(__cdb_days / 30) AS month,', + ' ROUND(__cdb_days / 7) AS week,', + ' __cdb_days AS day,', + ' __cdb_hours AS hour,', + ' __cdb_minutes AS minute,', + ' __cdb_seconds AS second', + 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' ].join('\n')); var MAX_INTERVAL_VALUE = 366; From b24858a17cb4d89c672573779ef7096131a38db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 11:21:44 +0200 Subject: [PATCH 082/128] Release 3.11.0 --- NEWS.md | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index bd89d748..1f5f64c7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog -## 3.10.2 -Released 2017-mm-dd +## 3.11.0 +Released 2017-08-08 Announcements: - Allow to override with any aggregation for histograms instantiated w/o aggregation. diff --git a/package.json b/package.json index df934d7a..898707a7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.10.2", + "version": "3.11.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 3611752677e8bdbef7a94f338c632d0a04ec67b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 11:25:18 +0200 Subject: [PATCH 083/128] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 1f5f64c7..bf1dd7e4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.11.1 +Released 2017-mm-dd + + ## 3.11.0 Released 2017-08-08 diff --git a/package.json b/package.json index 898707a7..a3e4416c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.11.0", + "version": "3.11.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From d27a2810671f45ab0ac1b75ab9f3a17535ad8c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 Aug 2017 17:55:39 +0200 Subject: [PATCH 084/128] Upgrade cartodb-redis to 0.14.0 --- package.json | 2 +- yarn.lock | 74 ++++++++++++++++++++++++++-------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 093a6948..212689cb 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "camshaft": "0.55.6", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", - "cartodb-redis": "cartodb/node-cartodb-redis#timeout-limits", + "cartodb-redis": "0.14.0", "debug": "~2.2.0", "dot": "~1.0.2", "express": "~4.13.3", diff --git a/yarn.lock b/yarn.lock index ebc2c0d0..74f6d935 100644 --- a/yarn.lock +++ b/yarn.lock @@ -223,6 +223,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -231,14 +239,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#0.15.1-cdb1": - version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -258,9 +258,9 @@ cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" -cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: - version "0.13.3" - resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/4c0586009dbfd464eb9e742bed39170334f40453" +cartodb-redis@0.14.0: + version "0.14.0" + resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.14.0.tgz#6f82fdb3e5b7c8005dbaccd6172c1706c4378df2" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" @@ -1741,7 +1741,32 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.81.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1768,31 +1793,6 @@ request@2.x, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" From ce97844f37e29816255e991548b65610946f3ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 Aug 2017 18:42:44 +0200 Subject: [PATCH 085/128] Upgrade windshaft version to 3.3.0 --- package.json | 2 +- yarn.lock | 108 +++++++++++++++++++++++++++------------------------ 2 files changed, 59 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index 212689cb..fe80d464 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#response-timeout-limit", + "windshaft": "3.3.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 74f6d935..e6aa6bce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -254,6 +254,14 @@ cartodb-psql@0.8.0: step "~0.0.6" underscore "~1.6.0" +cartodb-psql@0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.9.0.tgz#887ba006fb499ab789c15d10f34f6afbe1f2f68d" + dependencies: + debug "~2.2.0" + pg cartodb/node-postgres#6.1.2-cdb1 + underscore "~1.6.0" + cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" @@ -1312,7 +1320,7 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0: +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" @@ -1343,7 +1351,7 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: +node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36: version "0.6.36" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" dependencies: @@ -1520,7 +1528,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1741,32 +1749,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.81.0: +request@2.x, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1793,6 +1776,31 @@ request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1964,11 +1972,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.8" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.8.tgz#4cbcf965d8b901d1b1015cbc7fc415aae157dfaa" + version "3.1.9" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.9.tgz#b2e7fbaa348380318d3834323918c3c351b8bf18" dependencies: - nan "~2.4.0" - node-pre-gyp "~0.6.31" + nan "~2.6.2" + node-pre-gyp "~0.6.36" srs@1.x: version "1.2.0" @@ -2103,17 +2111,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#query-timeout: - version "2.3.1-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/38d5d82355aa9eef68c29b7d7a33b60a3ae394f7" +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": + version "2.3.1-cdb4" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#custom-timeout: - version "0.6.18-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/f9042e5eab92e26fb9e2a6ace65af92cc176a218" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": + version "0.6.18-cdb3" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2274,14 +2282,14 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#response-timeout-limit: - version "3.2.2" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/21ebe5f69b5b6215ec2f93088d8eee436f6b81d5" +windshaft@3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.0.tgz#bf8ef35f9a2bb126934531b0020f5e8f96b37dfa" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.8.0" + cartodb-psql "0.9.0" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" @@ -2293,8 +2301,8 @@ windshaft@cartodb/windshaft#response-timeout-limit: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#query-timeout - tilelive-mapnik cartodb/tilelive-mapnik#custom-timeout + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb4 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb3 torque.js "~2.11.0" underscore "~1.6.0" From ff560ffde7079d4e39c7b96cb95a33ae623e391d Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 9 Aug 2017 18:49:59 +0200 Subject: [PATCH 086/128] add test boundingBox-polygon-counter --- test/acceptance/dataviews/polygonCount.js | 79 +++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 test/acceptance/dataviews/polygonCount.js diff --git a/test/acceptance/dataviews/polygonCount.js b/test/acceptance/dataviews/polygonCount.js new file mode 100644 index 00000000..02ff075e --- /dev/null +++ b/test/acceptance/dataviews/polygonCount.js @@ -0,0 +1,79 @@ +require('../../support/test_helper'); +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + +describe('boundingBox-polygon-counter', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_formula: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: "cartodb_id", + operation: "count", + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": ` + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-161.015625,69.28725695167886],[-162.7734375,-7.710991655433217],[-40.78125,-8.059229627200192],[-161.015625,69.28725695167886]]]}'), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id + UNION ALL + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-29.179687499999996,-7.01366792756663],[103.71093749999999,-6.664607562172573],[105.46875,69.16255790810501],[-29.179687499999996,-7.01366792756663]]]}'), 4326), 3857), 2 + UNION ALL + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-117.42187500000001,68.13885164925573],[-35.859375,20.96143961409684],[59.4140625,68.52823492039876],[-117.42187500000001,68.13885164925573]]]}'), 4326), 3857), 3 + ` + } + } + ] + ); + + it('should not count the polygons outside the bounding box', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + params = { + bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' + } + this.testClient.getDataview('val_formula', params, function(err, dataview) { + assert.equal(dataview.result, 1); + done(); + }); + }); +}); \ No newline at end of file From 814b123b2bc594dd1aeddcaad1002af5aeeb1e8f Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 9 Aug 2017 18:55:14 +0200 Subject: [PATCH 087/128] fix 725 using the ST_Intersects function instead of && --- lib/cartodb/models/filter/bbox.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js index dcf19e55..0f7072ae 100644 --- a/lib/cartodb/models/filter/bbox.js +++ b/lib/cartodb/models/filter/bbox.js @@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([ ].join('\n')); var bboxFilterTpl = dot.template( - '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})' + 'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))' ); var LATITUDE_MAX_VALUE = 85.0511287798066; From 44c5eb051d692cd3c051d88812ccefa166a057a7 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 11:05:36 +0200 Subject: [PATCH 088/128] formatting the query of polygon count test --- test/acceptance/dataviews/polygonCount.js | 46 ++++++++++++++++++++--- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/test/acceptance/dataviews/polygonCount.js b/test/acceptance/dataviews/polygonCount.js index 02ff075e..02ceef28 100644 --- a/test/acceptance/dataviews/polygonCount.js +++ b/test/acceptance/dataviews/polygonCount.js @@ -21,6 +21,36 @@ describe('boundingBox-polygon-counter', function() { } }); + var polygon1 = { + type: "Polygon", + coordinates:[[ + [-161.015625,69.28725695167886], + [-162.7734375,-7.710991655433217], + [-40.78125,-8.059229627200192], + [-161.015625,69.28725695167886] + ]] + }; + + var polygon2 = { + type: "Polygon", + coordinates: [[ + [-29.179687499999996,-7.01366792756663], + [103.71093749999999,-6.664607562172573], + [105.46875,69.16255790810501], + [-29.179687499999996,-7.01366792756663] + ]] + }; + + var polygon3 = { + type: "Polygon", + coordinates:[[ + [-117.42187500000001,68.13885164925573], + [-35.859375,20.96143961409684], + [59.4140625,68.52823492039876], + [-117.42187500000001,68.13885164925573] + ]] + }; + var mapConfig = createMapConfig( [ { @@ -53,13 +83,19 @@ describe('boundingBox-polygon-counter', function() { "params": { "query": ` SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-161.015625,69.28725695167886],[-162.7734375,-7.710991655433217],[-40.78125,-8.059229627200192],[-161.015625,69.28725695167886]]]}'), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(polygon1)}' + ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id UNION ALL SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-29.179687499999996,-7.01366792756663],[103.71093749999999,-6.664607562172573],[105.46875,69.16255790810501],[-29.179687499999996,-7.01366792756663]]]}'), 4326), 3857), 2 + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(polygon2)}' + ), 4326), 3857), 2 UNION ALL SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-117.42187500000001,68.13885164925573],[-35.859375,20.96143961409684],[59.4140625,68.52823492039876],[-117.42187500000001,68.13885164925573]]]}'), 4326), 3857), 3 + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(polygon3)}' + ), 4326), 3857), 3 ` } } @@ -68,9 +104,9 @@ describe('boundingBox-polygon-counter', function() { it('should not count the polygons outside the bounding box', function(done) { this.testClient = new TestClient(mapConfig, 1234); - params = { + var params = { bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' - } + }; this.testClient.getDataview('val_formula', params, function(err, dataview) { assert.equal(dataview.result, 1); done(); From 23edf78a67e5c87753059bd3eeb989c5b063dbfd Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 15:58:25 +0200 Subject: [PATCH 089/128] Remove unnecessary step --- lib/cartodb/api/user_limits_api.js | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 74d0cf21..0f4f0f4c 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -23,23 +23,19 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) render: self.options.limits.render || 0 }; - step( - function getTimeoutLimit() { - self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { - if (err) { - return callback(err); - } - - if (timeoutRenderLimit && timeoutRenderLimit.render) { - if (Number.isFinite(timeoutRenderLimit.render)) { - limits.render = timeoutRenderLimit.render; - } - } - - return callback(null, limits); - }); + self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { + if (err) { + return callback(err); } - ); + + if (timeoutRenderLimit && timeoutRenderLimit.render) { + if (Number.isFinite(timeoutRenderLimit.render)) { + limits.render = timeoutRenderLimit.render; + } + } + + return callback(null, limits); + }); }; UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { From 69eaa72819a0efd0d9c0a8a1e9c0115cb6aca2fc Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:06:10 +0200 Subject: [PATCH 090/128] String comparison and regex to match errors instead of indexOf --- lib/cartodb/controllers/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 98c5ee2f..f74702e9 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -301,11 +301,11 @@ function statusFromErrorMessage(errMsg) { } function isRenderTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('Render timed out')); + return err.message === 'Render timed out'; } function isDatasourceTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); + return err.message && err.message.match(/canceling statement due to statement timeout/i); } function isTimeoutError (err) { From e7c206762dda88ea4e3b7322cad70ccca31da6e2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:09:26 +0200 Subject: [PATCH 091/128] String comparison and regex to match errors instead of indexOf --- lib/cartodb/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7aea23cd..d9302276 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -120,11 +120,11 @@ module.exports = function(serverOptions) { onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { function isRenderTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('Render timed out')); + return err.message === 'Render timed out'; } function isDatasourceTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); + return err.message && err.message.match(/canceling statement due to statement timeout/i); } function isTimeoutError (err) { From 14d5ee417881206bb7b73e60cfcdf8aab040e67a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:22:31 +0200 Subject: [PATCH 092/128] Remove user param --- .../acceptance/user-database-timeout-limit.js | 44 +++++++++---------- test/support/test-client.js | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index e1ff7bda..03a5e0c5 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -81,11 +81,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -118,11 +118,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -157,11 +157,11 @@ describe('user database timeout limit', function () { describe('fetching raster tiles', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); describe('with onTileErrorStrategy ENABLED', function () { @@ -326,11 +326,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -390,11 +390,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('"mvt" fails due to statement timeout', function (done) { @@ -428,11 +428,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -492,11 +492,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('"grid.json" fails due to statement timeout', function (done) { @@ -532,11 +532,11 @@ describe('user database timeout limit', function () { cartocss: TestClient.CARTOCSS.TORQUE }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -599,11 +599,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('"torque.json" fails due to statement timeout', function (done) { @@ -663,11 +663,11 @@ describe('user database timeout limit', function () { } }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -738,11 +738,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('fails due to statement timeout', function (done) { diff --git a/test/support/test-client.js b/test/support/test-client.js index 196e1a8e..3be8bf71 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1164,7 +1164,7 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) { +TestClient.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) From 95244334374d0fe31b7b2927d740294988926d1d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:24:40 +0200 Subject: [PATCH 093/128] Use instance server --- test/support/test-client.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 3be8bf71..1f8344df 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -487,7 +487,7 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -526,7 +526,7 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId; - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', From 01a22a45bb3973e86dba433773d2507286a958d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 10 Aug 2017 17:09:05 +0200 Subject: [PATCH 094/128] Move `setUserDatabaseTimeoutLimit` from class method to a instance method --- .../acceptance/user-database-timeout-limit.js | 88 +++++++++++-------- test/support/test-client.js | 2 +- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 03a5e0c5..42c468fd 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -81,11 +81,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -118,11 +118,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -156,14 +156,6 @@ describe('user database timeout limit', function () { describe('fetching raster tiles', function () { describe('with user\'s timeout of 200 ms', function () { - beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); - }); - - afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); - }); - describe('with onTileErrorStrategy ENABLED', function () { let onTileErrorStrategy; @@ -180,21 +172,31 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.setUserDatabaseTimeoutLimit(200, (err) => { if (err) { return done(err); } - this.layergroupid = res.layergroupid; + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - done(); + this.layergroupid = res.layergroupid; + + done(); + }); }); }); afterEach(function (done) { global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - - this.testClient.drain(done); + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('"png" fails due to statement timeout', function (done) { @@ -252,21 +254,33 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.setUserDatabaseTimeoutLimit(200, (err) => { if (err) { return done(err); } - this.layergroupid = res.layergroupid; + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - done(); + this.layergroupid = res.layergroupid; + + done(); + }); }); }); afterEach(function (done) { global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - this.testClient.drain(done); + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + + this.testClient.drain(done); + }); }); it('"png" fails due to statement timeout', function (done) { @@ -326,11 +340,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -390,11 +404,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('"mvt" fails due to statement timeout', function (done) { @@ -428,11 +442,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -492,11 +506,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('"grid.json" fails due to statement timeout', function (done) { @@ -532,11 +546,11 @@ describe('user database timeout limit', function () { cartocss: TestClient.CARTOCSS.TORQUE }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -599,11 +613,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('"torque.json" fails due to statement timeout', function (done) { @@ -663,11 +677,11 @@ describe('user database timeout limit', function () { } }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -738,11 +752,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('fails due to statement timeout', function (done) { diff --git a/test/support/test-client.js b/test/support/test-client.js index 1f8344df..21918665 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1164,7 +1164,7 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { +TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) From e1e22de65fe39da07924bbc441a74c54272c05fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 10 Aug 2017 17:52:46 +0200 Subject: [PATCH 095/128] Release 3.12.0 --- NEWS.md | 9 +++++++-- package.json | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index bf1dd7e4..2a7fdc46 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,12 @@ # Changelog -## 3.11.1 -Released 2017-mm-dd +## 3.12.0 +Released 2017-08-10 + +Announcements: + - Apply max tile response time for requests to layergoup, tiles, static maps, attributes and dataviews services #717. + - Upgrades windshaft to [3.3.0](https://github.com/CartoDB/windshaft/releases/tag/3.3.0). + - Upgrades cartodb-redis to [0.14.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/0.14.0). ## 3.11.0 diff --git a/package.json b/package.json index d0e0be6b..f482fffb 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.11.1", + "version": "3.12.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From e678957a8f0536abb744a99c767182eb2510c152 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 18:09:18 +0200 Subject: [PATCH 096/128] move polygon count test to widgets regression testfile, and check the only returned polygon is the expected one --- test/acceptance/dataviews/polygonCount.js | 115 ---------------------- test/acceptance/widgets/regressions.js | 97 ++++++++++++++++++ 2 files changed, 97 insertions(+), 115 deletions(-) delete mode 100644 test/acceptance/dataviews/polygonCount.js diff --git a/test/acceptance/dataviews/polygonCount.js b/test/acceptance/dataviews/polygonCount.js deleted file mode 100644 index 02ceef28..00000000 --- a/test/acceptance/dataviews/polygonCount.js +++ /dev/null @@ -1,115 +0,0 @@ -require('../../support/test_helper'); -var assert = require('../../support/assert'); -var TestClient = require('../../support/test-client'); - -function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; -} - -describe('boundingBox-polygon-counter', function() { - - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - - var polygon1 = { - type: "Polygon", - coordinates:[[ - [-161.015625,69.28725695167886], - [-162.7734375,-7.710991655433217], - [-40.78125,-8.059229627200192], - [-161.015625,69.28725695167886] - ]] - }; - - var polygon2 = { - type: "Polygon", - coordinates: [[ - [-29.179687499999996,-7.01366792756663], - [103.71093749999999,-6.664607562172573], - [105.46875,69.16255790810501], - [-29.179687499999996,-7.01366792756663] - ]] - }; - - var polygon3 = { - type: "Polygon", - coordinates:[[ - [-117.42187500000001,68.13885164925573], - [-35.859375,20.96143961409684], - [59.4140625,68.52823492039876], - [-117.42187500000001,68.13885164925573] - ]] - }; - - var mapConfig = createMapConfig( - [ - { - "type": "cartodb", - "options": { - "source": { - "id": "a0" - }, - "cartocss": "#points { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0" - } - } - ], - { - val_formula: { - source: { - id: 'a0' - }, - type: 'formula', - options: { - column: "cartodb_id", - operation: "count", - } - } - }, - [ - { - "id": "a0", - "type": "source", - "params": { - "query": ` - SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( - '${JSON.stringify(polygon1)}' - ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id - UNION ALL - SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( - '${JSON.stringify(polygon2)}' - ), 4326), 3857), 2 - UNION ALL - SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( - '${JSON.stringify(polygon3)}' - ), 4326), 3857), 3 - ` - } - } - ] - ); - - it('should not count the polygons outside the bounding box', function(done) { - this.testClient = new TestClient(mapConfig, 1234); - var params = { - bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' - }; - this.testClient.getDataview('val_formula', params, function(err, dataview) { - assert.equal(dataview.result, 1); - done(); - }); - }); -}); \ No newline at end of file diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 1d006ac4..f3ab3aa8 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -218,6 +218,103 @@ describe('widgets-regressions', function() { }); }); + + it('should not count the polygons outside the bounding box', function(done) { + var notIntersectingLeftTriangle = { + type: "Polygon", + coordinates:[[ + [-161.015625,69.28725695167886], + [-162.7734375,-7.710991655433217], + [-40.78125,-8.059229627200192], + [-161.015625,69.28725695167886] + ]] + }; + + var notIntersectingRightTriangle = { + type: "Polygon", + coordinates: [[ + [-29.179687499999996,-7.01366792756663], + [103.71093749999999,-6.664607562172573], + [105.46875,69.16255790810501], + [-29.179687499999996,-7.01366792756663] + ]] + }; + + var intersectingTriangle = { + type: "Polygon", + coordinates:[[ + [-117.42187500000001,68.13885164925573], + [-35.859375,20.96143961409684], + [59.4140625,68.52823492039876], + [-117.42187500000001,68.13885164925573] + ]] + }; + + let query = ` + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(notIntersectingLeftTriangle)}' + ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id, 'notIntersectingLeftTriangle' AS name + UNION + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(notIntersectingRightTriangle)}' + ), 4326), 3857), 2, 'notIntersectingRightTriangle' + UNION + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(intersectingTriangle)}' + ), 4326), 3857), 3, 'intersectingTriangle' + ` + + var mapConfig = { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + dataviews: { + val_formula: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: "name", + aggregation: "count", + } + } + }, + analyses: [ + { + "id": "a0", + "type": "source", + "params": { + "query": query + } + } + ] + }; + + this.testClient = new TestClient(mapConfig, 1234); + var params = { + bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' + }; + this.testClient.getDataview('val_formula', params, function(err, dataview) { + assert.equal(dataview.categories.length, 1); + assert.equal(dataview.categories[0].category, 'intersectingTriangle') + done(); + }); + }); + }); }); From fab7832deedb322d68c6270abc136c45a64cfae7 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 18:16:53 +0200 Subject: [PATCH 097/128] added ascii art for polygons count test --- test/acceptance/widgets/regressions.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index f3ab3aa8..1527c8ae 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -220,6 +220,16 @@ describe('widgets-regressions', function() { it('should not count the polygons outside the bounding box', function(done) { + + // $ % $ = not intersecting left triangle + // $$ **VVVVV** %% % = not intersecting right triangle + // $$$ *VVVVV* %%% * = intersecting triangle + // $$$$ ***** %%%% V = bounding box + // $$$$$ *** %%%%% + // $$$$$$ * %%%%%% + // $$$$$$$ %%%%%%% + // $$$$$$$$ %%%%%%%% + var notIntersectingLeftTriangle = { type: "Polygon", coordinates:[[ From 490adbce4bf316c19e4acd129a2cb9d4eefc5fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 10 Aug 2017 18:19:20 +0200 Subject: [PATCH 098/128] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2a7fdc46..d82fa076 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.1 +Released 2017-mm-dd + + ## 3.12.0 Released 2017-08-10 diff --git a/package.json b/package.json index f482fffb..9ca7ab08 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.0", + "version": "3.12.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 92d33bf7fd7a94530e39bdd1b7cc5f6eacad8236 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 18:20:15 +0200 Subject: [PATCH 099/128] linter details for polygons count test --- test/acceptance/widgets/regressions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 1527c8ae..1c2e8f45 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -275,7 +275,7 @@ describe('widgets-regressions', function() { ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( '${JSON.stringify(intersectingTriangle)}' ), 4326), 3857), 3, 'intersectingTriangle' - ` + `; var mapConfig = { version: '1.5.0', @@ -320,7 +320,7 @@ describe('widgets-regressions', function() { }; this.testClient.getDataview('val_formula', params, function(err, dataview) { assert.equal(dataview.categories.length, 1); - assert.equal(dataview.categories[0].category, 'intersectingTriangle') + assert.equal(dataview.categories[0].category, 'intersectingTriangle'); done(); }); }); From d0f5ebd7ab8f3700e9a314bf21273879f3f0c38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 Aug 2017 17:55:55 +0200 Subject: [PATCH 100/128] Restore statsClient after performing test --- test/unit/cartodb/ported/tile_stats.test.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit/cartodb/ported/tile_stats.test.js b/test/unit/cartodb/ported/tile_stats.test.js index 45b1fa29..b71f1384 100644 --- a/test/unit/cartodb/ported/tile_stats.test.js +++ b/test/unit/cartodb/ported/tile_stats.test.js @@ -6,10 +6,13 @@ var LayergroupController = require('../../../../lib/cartodb/controllers/layergro describe('tile stats', function() { - after(function() { - global.statsClient = null; + beforeEach(function () { + this.statsClient = global.statsClient; }); + afterEach(function() { + global.statsClient = this.statsClient; + }); it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() { var expectedCalls = 2, // it will call increment once for the general error From 7be74d6ce1534f00a6bced6428902a7bf02c67ea Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Sun, 13 Aug 2017 17:40:06 +0200 Subject: [PATCH 101/128] Upgrades cartodb-psql, windshaft, and camshaft --- NEWS.md | 3 + package.json | 6 +- yarn.lock | 225 +++++++++++++++++++++++++-------------------------- 3 files changed, 118 insertions(+), 116 deletions(-) diff --git a/NEWS.md b/NEWS.md index d82fa076..114d4b43 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,9 @@ ## 3.12.1 Released 2017-mm-dd + - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1). + - Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1). + - Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7). ## 3.12.0 diff --git a/package.json b/package.json index 9ca7ab08..421fa75c 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.6", - "cartodb-psql": "0.8.0", + "camshaft": "0.55.7", + "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", "debug": "~2.2.0", @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.3.0", + "windshaft": "3.3.1", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index bf5e11e4..dc627c40 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -109,9 +109,9 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -balanced-match@^0.4.1: - version "0.4.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -120,8 +120,8 @@ bcrypt-pbkdf@^1.0.0: tweetnacl "^0.14.3" bindings@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.2.1.tgz#14ad6113812d2d37d72e67b4cacb4bb726505f11" + version "1.3.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7" block-stream@*: version "0.0.9" @@ -150,11 +150,11 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.0.0: - version "1.1.7" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" +brace-expansion@^1.1.7: + version "1.1.8" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" dependencies: - balanced-match "^0.4.1" + balanced-match "^1.0.0" concat-map "0.0.1" browser-stdout@1.3.0: @@ -194,18 +194,18 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.6: - version "0.55.6" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.6.tgz#11af28051c3b911fb023ae1cafb165bbd040f174" +camshaft@0.55.7: + version "0.55.7" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.7.tgz#6f09a0e9618a576ce89946e259189d6adf0c9b51" dependencies: async "^1.5.2" bunyan "1.8.1" - cartodb-psql "0.8.0" + cartodb-psql "^0.10.1" debug "^2.2.0" dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@0.16.3: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -245,21 +245,12 @@ cartocolor@4.0.0: dependencies: colorbrewer "1.0.0" -cartodb-psql@0.8.0: - version "0.8.0" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" +cartodb-psql@0.10.1, cartodb-psql@^0.10.1: + version "0.10.1" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.1.tgz#0ac947e62fe10b27916df6b7ba6c461953fe3a23" dependencies: debug "~2.2.0" - pg cartodb/node-postgres#6.1.2-cdb1 - step "~0.0.6" - underscore "~1.6.0" - -cartodb-psql@0.9.0: - version "0.9.0" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.9.0.tgz#887ba006fb499ab789c15d10f34f6afbe1f2f68d" - dependencies: - debug "~2.2.0" - pg cartodb/node-postgres#6.1.2-cdb1 + pg cartodb/node-postgres#6.1.6-cdb1 underscore "~1.6.0" cartodb-query-tables@0.2.0: @@ -388,7 +379,7 @@ cookie@0.1.5: version "0.1.5" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.1.5.tgz#6ab9948a4b1ae21952cd2588530a4722d4044d7c" -core-util-is@~1.0.0: +core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -425,10 +416,10 @@ debug@2.6.0: ms "0.7.2" debug@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" + version "1.0.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" dependencies: - ms "0.6.2" + ms "2.0.0" decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" @@ -461,8 +452,8 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" depd@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.0.tgz#e1bd82c6aab6ced965b97b88b17ed3e528ca18c3" + version "1.1.1" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" destroy@~1.0.4: version "1.0.4" @@ -561,9 +552,9 @@ esprima@2.7.x, esprima@^2.7.1: version "2.7.3" resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" -esprima@^3.1.1: - version "3.1.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" +esprima@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" estraverse@^1.9.1: version "1.9.3" @@ -615,9 +606,9 @@ extend@~3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" -extsprintf@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.0.2.tgz#e1080e0658e300b06294990cc70e1502235fd550" +extsprintf@1.3.0, extsprintf@^1.2.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" fast-levenshtein@~2.0.4: version "2.0.6" @@ -716,7 +707,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: +generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -728,6 +719,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" +generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.6" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -796,8 +791,8 @@ growl@1.9.2: resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: - version "4.0.8" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" + version "4.0.10" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -860,8 +855,8 @@ hoek@2.x.x: resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" hosted-git-info@^2.1.4: - version "2.4.2" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.2.tgz#0076b9f46a270506ddbaaea56496897460612a67" + version "2.5.0" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" htmlparser2@3.8.x: version "3.8.3" @@ -995,30 +990,28 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" -jodid25519@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" - dependencies: - jsbn "~0.1.0" - js-base64@^2.1.9: version "2.1.9" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" +js-string-escape@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" + js-yaml@3.x, js-yaml@^3.4.6: - version "3.8.4" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.4.tgz#520b4564f86573ba96662af85a8cafa7b4b5a6f6" + version "3.9.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.9.1.tgz#08775cebdfdd359209f0d2acd383c8f86a6904a0" dependencies: argparse "^1.0.7" - esprima "^3.1.1" + esprima "^4.0.0" jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" jshint@~2.9.4: - version "2.9.4" - resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.4.tgz#5e3ba97848d5290273db514aee47fe24cf592934" + version "2.9.5" + resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.5.tgz#1e7252915ce681b40827ee14248c46d34e9aa62c" dependencies: cli "~1.0.0" console-browserify "1.1.x" @@ -1056,13 +1049,13 @@ jsonpointer@^4.0.0: resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-4.0.1.tgz#4fd92cb34e0e9db3c89c8622ecf51f9b978c6cb9" jsprim@^1.2.2: - version "1.4.0" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.0.tgz#a3b87e40298d8c380552d8cc7628a0bb95a22918" + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" dependencies: assert-plus "1.0.0" - extsprintf "1.0.2" + extsprintf "1.3.0" json-schema "0.2.3" - verror "1.3.6" + verror "1.10.0" kind-of@^3.0.2: version "3.2.2" @@ -1238,15 +1231,15 @@ millstone@0.6.17: underscore "~1.6.0" zipfile "~0.5.5" -mime-db@~1.27.0: - version "1.27.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1" +mime-db@~1.29.0: + version "1.29.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.29.0.tgz#48d26d235589651704ac5916ca06001914266878" mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: - version "2.1.15" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed" + version "2.1.16" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.16.tgz#2b858a52e5ecd516db897ac2be87487830698e23" dependencies: - mime-db "~1.27.0" + mime-db "~1.29.0" mime@1.3.4, mime@~1.3.4: version "1.3.4" @@ -1257,10 +1250,10 @@ mime@~1.2.11: resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: - brace-expansion "^1.0.0" + brace-expansion "^1.1.7" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1281,8 +1274,8 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi minimist "0.0.8" mocha@~3.4.1: - version "3.4.1" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.1.tgz#a3802b4aa381934cacb38de70cf771621da8f9af" + version "3.4.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" dependencies: browser-stdout "1.3.0" commander "2.9.0" @@ -1300,10 +1293,6 @@ moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" -ms@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.6.2.tgz#d89c2124c6fdc1353d65a8b77bf1aac4b193708c" - ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" @@ -1312,6 +1301,10 @@ ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1383,8 +1376,8 @@ nopt@^4.0.1: osenv "^0.1.4" normalize-package-data@^2.3.2: - version "2.3.8" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.8.tgz#d819eda2a9dedbd1ffa563ea4071d936782295bb" + version "2.4.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" dependencies: hosted-git-info "^2.1.4" is-builtin-module "^1.0.0" @@ -1408,10 +1401,14 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1512,27 +1509,28 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.7.1" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" + version "1.8.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" dependencies: generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: - version "1.11.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.11.0.tgz#aae91a82d952b633bb88d006350a166daaf6ea90" + version "1.12.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" dependencies: ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" - postgres-interval "~1.0.0" + postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": - version "6.1.2" - resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" +"pg@github:cartodb/node-postgres#6.1.6-cdb1": + version "6.1.6" + resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: buffer-writer "1.0.1" + js-string-escape "1.0.1" packet-reader "0.2.0" pg-connection-string "0.1.3" pg-pool "1.*" @@ -1541,8 +1539,8 @@ pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1 semver "4.3.2" pgpass@1.x: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.1.tgz#0de8b5bef993295d90a7e17d976f568dcd25d49f" + version "1.0.2" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" dependencies: split "^1.0.0" @@ -1605,9 +1603,9 @@ postgres-date@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" -postgres-interval@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.0.2.tgz#7261438d862b412921c6fdb7617668424b73a6ed" +postgres-interval@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.1.tgz#acdb0f897b4b1c6e496d9d4e0a853e1c428f06f0" dependencies: xtend "^4.0.0" @@ -1840,8 +1838,8 @@ safe-json-stringify@~1: resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" "semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" semver@4.3.2: version "4.3.2" @@ -1962,8 +1960,8 @@ sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphe resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" split@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/split/-/split-1.0.0.tgz#c4395ce683abcd254bc28fe1dabb6e5c27dcffae" + version "1.0.1" + resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" dependencies: through "2" @@ -1985,8 +1983,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" + version "1.13.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -1995,7 +1993,6 @@ sshpk@^1.7.0: optionalDependencies: bcrypt-pbkdf "^1.0.0" ecc-jsbn "~0.1.1" - jodid25519 "^1.0.0" jsbn "~0.1.0" tweetnacl "~0.14.0" @@ -2197,8 +2194,8 @@ type-is@~1.6.10, type-is@~1.6.6: mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.26" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.26.tgz#3a1db8ae0a0aba7f92e1ddadadbd0293d549f90e" + version "2.8.29" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -2238,8 +2235,8 @@ utils-merge@1.0.0: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" uuid@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" validate-npm-package-license@^3.0.1: version "3.0.1" @@ -2252,19 +2249,21 @@ vary@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/vary/-/vary-1.0.1.tgz#99e4981566a286118dfb2b817357df7993376d10" -verror@1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.3.6.tgz#cff5df12946d297d2baaefaa2689e25be01c005c" +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" dependencies: - extsprintf "1.0.2" + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" which-module@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" which@^1.1.1: - version "1.2.14" - resolved "https://registry.yarnpkg.com/which/-/which-1.2.14.tgz#9a87c4378f03e827cecaf1acdf56c736c01c14e5" + version "1.3.0" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" dependencies: isexe "^2.0.0" @@ -2282,14 +2281,14 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.0.tgz#bf8ef35f9a2bb126934531b0020f5e8f96b37dfa" +windshaft@3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.1.tgz#b5557fa6b0cfa13920904f57206b86f7aa054f74" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.9.0" + cartodb-psql "^0.10.1" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" From fef0dc302a6a534c009062f2f92c9a425b7f5d18 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Sun, 13 Aug 2017 17:55:45 +0200 Subject: [PATCH 102/128] Release 3.12.1 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 114d4b43..5f3fd700 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.1 -Released 2017-mm-dd +Released 2017-08-13 - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1). - Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1). - Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7). From 339f1aafa907d9f83126cbd5829b849468141731 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Sun, 13 Aug 2017 17:56:23 +0200 Subject: [PATCH 103/128] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5f3fd700..2a392e06 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.2 +Released 2017-mm-dd + + ## 3.12.1 Released 2017-08-13 - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1). diff --git a/package.json b/package.json index 421fa75c..8b3f1ae1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.1", + "version": "3.12.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 175d070f09b412179f883a20739cfbae7cf6e40d Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 16 Aug 2017 10:07:27 +0200 Subject: [PATCH 104/128] using const instead of let and var and adding assert.ifError --- test/acceptance/widgets/regressions.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 1c2e8f45..74f4544c 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -230,7 +230,7 @@ describe('widgets-regressions', function() { // $$$$$$$ %%%%%%% // $$$$$$$$ %%%%%%%% - var notIntersectingLeftTriangle = { + const notIntersectingLeftTriangle = { type: "Polygon", coordinates:[[ [-161.015625,69.28725695167886], @@ -240,7 +240,7 @@ describe('widgets-regressions', function() { ]] }; - var notIntersectingRightTriangle = { + const notIntersectingRightTriangle = { type: "Polygon", coordinates: [[ [-29.179687499999996,-7.01366792756663], @@ -250,7 +250,7 @@ describe('widgets-regressions', function() { ]] }; - var intersectingTriangle = { + const intersectingTriangle = { type: "Polygon", coordinates:[[ [-117.42187500000001,68.13885164925573], @@ -260,7 +260,7 @@ describe('widgets-regressions', function() { ]] }; - let query = ` + const query = ` SELECT ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( '${JSON.stringify(notIntersectingLeftTriangle)}' @@ -277,7 +277,7 @@ describe('widgets-regressions', function() { ), 4326), 3857), 3, 'intersectingTriangle' `; - var mapConfig = { + const mapConfig = { version: '1.5.0', layers: [ { @@ -315,10 +315,11 @@ describe('widgets-regressions', function() { }; this.testClient = new TestClient(mapConfig, 1234); - var params = { + const params = { bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' }; this.testClient.getDataview('val_formula', params, function(err, dataview) { + assert.ifError(err); assert.equal(dataview.categories.length, 1); assert.equal(dataview.categories[0].category, 'intersectingTriangle'); done(); From 3a22adf966751ab1ad5815233489635adf8d473d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 16 Aug 2017 15:35:34 +0200 Subject: [PATCH 105/128] Update news --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 2a392e06..436945cc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.12.2 Released 2017-mm-dd +Bug fixes: + - Polygon count problems #725. + ## 3.12.1 Released 2017-08-13 From f3b7a857f2c507e6d575c2fbc45f7b4585eb4a93 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 16 Aug 2017 15:36:16 +0200 Subject: [PATCH 106/128] Release 3.12.2 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 436945cc..982f164f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.2 -Released 2017-mm-dd +Released 2017-08-16 Bug fixes: - Polygon count problems #725. From 0ed46930ddf7d5d01d04210c125a8e487e1dbd39 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 16 Aug 2017 15:37:29 +0200 Subject: [PATCH 107/128] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 982f164f..46655deb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.3 +Released 2017-mm-dd + + ## 3.12.2 Released 2017-08-16 diff --git a/package.json b/package.json index 8b3f1ae1..3ae6fbb4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.2", + "version": "3.12.3", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 4491fa2fafa172e2e497cb59cd546da3f7a3815f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Aug 2017 14:17:17 +0200 Subject: [PATCH 108/128] Upgrade camshaft to 0.55.8 --- NEWS.md | 2 ++ package.json | 2 +- yarn.lock | 22 +++++++++++----------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/NEWS.md b/NEWS.md index 46655deb..ca885912 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.12.3 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8). ## 3.12.2 Released 2017-08-16 diff --git a/package.json b/package.json index 3ae6fbb4..26cc7847 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.7", + "camshaft": "0.55.8", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index dc627c40..f47cd66f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.7: - version "0.55.7" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.7.tgz#6f09a0e9618a576ce89946e259189d6adf0c9b51" +camshaft@0.55.8: + version "0.55.8" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.8.tgz#a289ee02598a764144d5a56fcceae964b425d63c" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -205,7 +205,7 @@ camshaft@0.55.7: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1525,7 +1525,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -2108,7 +2108,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2116,7 +2116,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: From 7aefca3f82656eb6a48e62999997de5b398dba67 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 22 Aug 2017 16:26:36 +0200 Subject: [PATCH 109/128] updating NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ca885912..66f95e6f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.3 -Released 2017-mm-dd +Released 2017-08-22 Announcements: - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8). From cd27d6aa02422acb1fff125846df52f3c6d64b86 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 22 Aug 2017 16:37:23 +0200 Subject: [PATCH 110/128] Stub NEWS/package for next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 66f95e6f..1d1c2caa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.4 +Released 2017-mm-dd + + ## 3.12.3 Released 2017-08-22 diff --git a/package.json b/package.json index 26cc7847..a4bddced 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.3", + "version": "3.12.4", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From ca0b927f517ea59baa7327748a3fdf9fe0154dc5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 23 Aug 2017 16:34:42 +0200 Subject: [PATCH 111/128] Update camshaft to 0.56.0 (#731) --- NEWS.md | 4 +++- package.json | 2 +- yarn.lock | 58 ++++++++++++++++++++++++++-------------------------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1d1c2caa..828b4081 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,10 @@ # Changelog ## 3.12.4 -Released 2017-mm-dd +Released 2017-08-23 +Announcements: + - Upgrades camshaft to [0.56.0](https://github.com/CartoDB/camshaft/releases/tag/0.56.0). ## 3.12.3 Released 2017-08-22 diff --git a/package.json b/package.json index a4bddced..462d9bdd 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.8", + "camshaft": "0.56.0", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index f47cd66f..8876dea5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.8: - version "0.55.8" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.8.tgz#a289ee02598a764144d5a56fcceae964b425d63c" +camshaft@0.56.0: + version "0.56.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.56.0.tgz#6adc8ae8e5b63638551ccca97cba8273642e4f1a" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -1747,7 +1747,32 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.81.0: +request@2.x, request@^2.55.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1774,31 +1799,6 @@ request@2.x, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" From ed0bfa5f63c35807a3917864bde07f7ea60c2248 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 23 Aug 2017 16:53:01 +0200 Subject: [PATCH 112/128] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 828b4081..3789a8ea 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.5 +Released 2017-mm-dd + + ## 3.12.4 Released 2017-08-23 diff --git a/package.json b/package.json index 462d9bdd..d5da7a77 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.4", + "version": "3.12.5", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From d644376f88d6c4d0e23ed24638d0089b1e242c10 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 24 Aug 2017 12:07:43 +0200 Subject: [PATCH 113/128] Update to camshaft 0.57.0 (#733) * Update to camshaft 0.57.0 * Yarn.lock with update to camshaft 0.57.0 --- NEWS.md | 4 +++- package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3789a8ea..5f59077f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,9 @@ # Changelog ## 3.12.5 -Released 2017-mm-dd +Released 2017-08-24 + + - Upgrades camshaft to [0.57.0](https://github.com/CartoDB/camshaft/releases/tag/0.57.0). ## 3.12.4 diff --git a/package.json b/package.json index d5da7a77..78283248 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.56.0", + "camshaft": "0.57.0", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index 8876dea5..e9c60d7c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.56.0: - version "0.56.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.56.0.tgz#6adc8ae8e5b63638551ccca97cba8273642e4f1a" +camshaft@0.57.0: + version "0.57.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.57.0.tgz#22c2ab92903e43d5af6b763870839fbcfe82c650" dependencies: async "^1.5.2" bunyan "1.8.1" From 03dc2601048d248eed79912ba04c4d114790c6a3 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 24 Aug 2017 12:09:51 +0200 Subject: [PATCH 114/128] Stubs new version --- NEWS.md | 5 +++++ package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5f59077f..79a2bdd4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 3.12.6 +Released 2017-mm-dd + + + ## 3.12.5 Released 2017-08-24 diff --git a/package.json b/package.json index 78283248..0a5d3621 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.5", + "version": "3.12.6", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 77f1aa7e0c845e01d22d6f4adf10d56d2bbf61e5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 31 Aug 2017 10:12:58 +0200 Subject: [PATCH 115/128] Update to camshaft 0.58.0 (#737) --- NEWS.md | 3 ++- package.json | 6 ++++-- yarn.lock | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 79a2bdd4..673eb60a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,9 @@ # Changelog ## 3.12.6 -Released 2017-mm-dd +Released 2017-08-31 + - Upgrades camshaft to [0.58.0](https://github.com/CartoDB/camshaft/releases/tag/0.58.0). ## 3.12.5 diff --git a/package.json b/package.json index 0a5d3621..a52a5f74 100644 --- a/package.json +++ b/package.json @@ -17,11 +17,13 @@ "Simon Tokumine ", "Javi Santana ", "Sandro Santilli ", - "Carlos Matallín " + "Carlos Matallín ", + "Daniel Garcia Aubert ", + "Mario de Frutos " ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.57.0", + "camshaft": "0.58.0", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index e9c60d7c..475b24ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.57.0: - version "0.57.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.57.0.tgz#22c2ab92903e43d5af6b763870839fbcfe82c650" +camshaft@0.58.0: + version "0.58.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.0.tgz#c352fd1f8fb84a85f580b84df79e9dfb556a0c9c" dependencies: async "^1.5.2" bunyan "1.8.1" From 5332fd3baa7602b228aeb6a398d01f589c129c4d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 31 Aug 2017 10:21:00 +0200 Subject: [PATCH 116/128] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 673eb60a..c37852a1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.7 +Released 2017-mm-dd + + ## 3.12.6 Released 2017-08-31 diff --git a/package.json b/package.json index a52a5f74..3321a45b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.6", + "version": "3.12.7", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 7be5361433c290e86fc881259708f74872cdd38e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 1 Sep 2017 11:31:06 +0200 Subject: [PATCH 117/128] Upgrade to camshaft 0.58.1 (#739) --- NEWS.md | 4 +++- package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index c37852a1..62e7129c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,9 @@ # Changelog ## 3.12.7 -Released 2017-mm-dd +Released 2017-09-01 + + - Upgrades camshaft to [0.58.1](https://github.com/CartoDB/camshaft/releases/tag/0.58.1). ## 3.12.6 diff --git a/package.json b/package.json index 3321a45b..058d0841 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.58.0", + "camshaft": "0.58.1", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index 475b24ad..8270d25a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.58.0: - version "0.58.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.0.tgz#c352fd1f8fb84a85f580b84df79e9dfb556a0c9c" +camshaft@0.58.1: + version "0.58.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.1.tgz#e4156580683f624212ea3020e59790ad006f24cc" dependencies: async "^1.5.2" bunyan "1.8.1" From bd8d147a7d511cacd162d1777cc8aea708312d33 Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Wed, 6 Sep 2017 16:21:01 +0200 Subject: [PATCH 118/128] Fix out of range bug in date histograms --- lib/cartodb/models/dataview/histogram.js | 4 ++-- test/acceptance/dataviews/histogram.js | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c27c696f..6e62ad9d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -198,8 +198,8 @@ var dateBasicsQueryTpl = dot.template([ var dateOverrideBasicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._end}}) AS __cdb_max_val,', - ' min({{=it._start}}) AS __cdb_min_val,', + ' max({{=it._end}})::float AS __cdb_max_val,', + ' min({{=it._start}})::float AS __cdb_min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(', ' date_trunc(', diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 3baf002b..c9e16489 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -332,6 +332,21 @@ describe('histogram-dataview for date column type', function() { }); }); + it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) { + var params = { + start: -2145916800, + end: 1009843199 + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + + done(); + }); + }); it('should return same histogram ' + test.desc, function (done) { var params = { From d77075295e1609ac1efed61dd255280d8a4bca69 Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Thu, 7 Sep 2017 10:48:21 +0200 Subject: [PATCH 119/128] Update version and NEWS --- NEWS.md | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 62e7129c..45a7e000 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.12.8 +Released 2017-mm-dd + +Bug fixes: +- Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962) + ## 3.12.7 Released 2017-09-01 diff --git a/package.json b/package.json index 058d0841..020438ee 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.7", + "version": "3.12.8", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 806c13beac1eb54700baf06d13e36b86b1833511 Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Thu, 7 Sep 2017 10:51:23 +0200 Subject: [PATCH 120/128] Release 3.12.8 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 45a7e000..a5436d6e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.8 -Released 2017-mm-dd +Released 2017-09-07 Bug fixes: - Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962) From 7dd5c5b15d178ead67609e797e949fcb1479380b Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 14:39:25 +0200 Subject: [PATCH 121/128] Do not use distinct when calculating quantiles --- lib/cartodb/backends/turbo-carto-postgres-datasource.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/turbo-carto-postgres-datasource.js b/lib/cartodb/backends/turbo-carto-postgres-datasource.js index 65109325..b2550eb7 100644 --- a/lib/cartodb/backends/turbo-carto-postgres-datasource.js +++ b/lib/cartodb/backends/turbo-carto-postgres-datasource.js @@ -21,10 +21,10 @@ function createTemplate(method) { } var methods = { - quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles', + quantiles: 'CDB_QuantileBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as quantiles', equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal', - jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks', - headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails' + jenks: 'CDB_JenksBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as jenks', + headtails: 'CDB_HeadsTailsBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as headtails' }; var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) { From f1246cb060178ce3abccc036e7d9f8ce20f5af91 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 15:53:59 +0200 Subject: [PATCH 122/128] Bump to 3.12.9 --- NEWS.md | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index a5436d6e..54ca402c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.12.9 +Released 2017-mm-dd + +Bug fixes: +- Do not use distinct when calculating quantiles. #743 + ## 3.12.8 Released 2017-09-07 diff --git a/package.json b/package.json index 020438ee..02e0d5ca 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.8", + "version": "3.12.9", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From a24792f46d140479cf051ab79fee22ad9a5007ec Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 15:57:11 +0200 Subject: [PATCH 123/128] Release 3.12.9 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 54ca402c..d2cbc809 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.9 -Released 2017-mm-dd +Released 2017-09-07 Bug fixes: - Do not use distinct when calculating quantiles. #743 From 855e5c9e4cb9d78b9a9eaa4ceb6564df4cc5e0d3 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 15:59:32 +0200 Subject: [PATCH 124/128] Stub next release --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index d2cbc809..15720ece 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 3.12.10 +Released 2017-mm-dd + ## 3.12.9 Released 2017-09-07 From 6b2e2b2241dea41d39cff82e52b124af17041336 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 16:00:27 +0200 Subject: [PATCH 125/128] Stub next release (package.json) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 02e0d5ca..12430490 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.9", + "version": "3.12.10", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From a5d9bfa0ec261abe178776c4140fa1e0ffacee57 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Sep 2017 10:34:04 +0000 Subject: [PATCH 126/128] Upgrades windshaft to 3.3.2 --- NEWS.md | 1 + package.json | 2 +- yarn.lock | 359 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 265 insertions(+), 97 deletions(-) diff --git a/NEWS.md b/NEWS.md index 15720ece..0aad89fd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## 3.12.10 Released 2017-mm-dd + - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). ## 3.12.9 Released 2017-09-07 diff --git a/package.json b/package.json index 12430490..7e4f4b52 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.3.1", + "windshaft": "3.3.2", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 8270d25a..a9ba816b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -48,10 +52,6 @@ ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" -ap@~0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" - aproba@^1.0.3: version "1.1.2" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" @@ -205,7 +205,7 @@ camshaft@0.58.1: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -343,12 +343,16 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0, commander@^2.9.0: +commander@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" +commander@^2.9.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -368,8 +372,8 @@ content-disposition@0.5.1: resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.1.tgz#87476c6a67c8daa87e32e87616df883ba7fb071b" content-type@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.2.tgz#b7d113aee7a8dd27bd21133c4dc2529df1721eed" + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" cookie-signature@1.0.6: version "1.0.6" @@ -403,7 +407,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@^2.2.0, debug@~2.2.0: +debug@2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -421,6 +425,12 @@ debug@^1.0.4: dependencies: ms "2.0.0" +debug@^2.2.0: + version "2.6.8" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" + dependencies: + ms "2.0.0" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -431,7 +441,7 @@ deep-eql@^0.1.3: dependencies: type-detect "0.1.1" -deep-equal@^1.0.0: +deep-equal@^1.0.0, deep-equal@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" @@ -443,6 +453,17 @@ deep-is@~0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" +define-properties@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94" + dependencies: + foreach "^2.0.5" + object-keys "^1.0.8" + +defined@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -491,7 +512,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -525,6 +550,24 @@ error-ex@^1.2.0: dependencies: is-arrayish "^0.2.1" +es-abstract@^1.5.0: + version "1.8.2" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.8.2.tgz#25103263dc4decbda60e0c737ca32313518027ee" + dependencies: + es-to-primitive "^1.1.1" + function-bind "^1.1.1" + has "^1.0.1" + is-callable "^1.1.3" + is-regex "^1.0.4" + +es-to-primitive@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.1.1.tgz#45355248a88979034b6792e19bb81f2b7975dd0d" + dependencies: + is-callable "^1.1.1" + is-date-object "^1.0.1" + is-symbol "^1.0.1" + es6-promise@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.1.2.tgz#795e25ceb47f7babb263d151afbedd92d18e6a07" @@ -636,6 +679,16 @@ find-up@^1.0.0: path-exists "^2.0.0" pinkie-promise "^2.0.0" +for-each@~0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.2.tgz#2c40450b9348e97f281322593ba96704b9abd4d4" + dependencies: + is-function "~1.0.0" + +foreach@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -649,8 +702,8 @@ form-data@~2.1.1: mime-types "^2.1.12" forwarded@~0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.0.tgz#19ef9874c4ae1c297bcf078fde63a09b66a84363" + version "0.1.2" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" fresh@0.3.0: version "0.3.0" @@ -677,6 +730,10 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: mkdirp ">=0.5 0" rimraf "2" +function-bind@^1.0.2, function-bind@^1.1.1, function-bind@~1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + gauge@~2.7.3: version "2.7.4" resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" @@ -691,11 +748,11 @@ gauge@~2.7.3: wide-align "^1.1.0" gdal@~0.9.2: - version "0.9.4" - resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.4.tgz#6dad4abb8ffb3e0d51150fb7935ad7c622c81818" + version "0.9.6" + resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.6.tgz#0cf75d830d35847b4274368b10e04a925321a0ba" dependencies: - nan "~2.5.0" - node-pre-gyp "~0.6.27" + nan "~2.6.2" + node-pre-gyp "~0.6.36" generate-function@^2.0.0: version "2.0.0" @@ -733,7 +790,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -764,6 +821,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5, glob@^7.1.1, glob@~7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -834,6 +902,12 @@ has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" +has@^1.0.1, has@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.1.tgz#8461733f538b0837c9361e39a9ab9e9704dc2f28" + dependencies: + function-bind "^1.0.2" + hawk@~3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" @@ -928,15 +1002,27 @@ is-builtin-module@^1.0.0: dependencies: builtin-modules "^1.0.0" +is-callable@^1.1.1, is-callable@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.3.tgz#86eb75392805ddc33af71c92a0eedf74ee7604b2" + +is-date-object@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.1.tgz#9aa20eb6aeebbff77fbd33e74ca01b33581d3a16" + is-fullwidth-code-point@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" dependencies: number-is-nan "^1.0.0" +is-function@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-function/-/is-function-1.0.1.tgz#12cfb98b65b57dd3d193a3121f5f6e2f437602b5" + is-my-json-valid@^2.12.4: - version "2.16.0" - resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.0.tgz#f079dd9bfdae65ee2038aae8acbc86ab109e3693" + version "2.16.1" + resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11" dependencies: generate-function "^2.0.0" generate-object-property "^1.1.0" @@ -947,6 +1033,16 @@ is-property@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84" +is-regex@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491" + dependencies: + has "^1.0.1" + +is-symbol@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.1.tgz#3cc59f00025194b6ab2e38dbae6689256b660572" + is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -991,16 +1087,16 @@ istanbul@~0.4.3: wordwrap "^1.0.0" js-base64@^2.1.9: - version "2.1.9" - resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" + version "2.3.2" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.3.2.tgz#a79a923666372b580f8e27f51845c6f7e8fbfbaf" js-string-escape@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" js-yaml@3.x, js-yaml@^3.4.6: - version "3.9.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.9.1.tgz#08775cebdfdd359209f0d2acd383c8f86a6904a0" + version "3.10.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" dependencies: argparse "^1.0.7" esprima "^4.0.0" @@ -1231,17 +1327,17 @@ millstone@0.6.17: underscore "~1.6.0" zipfile "~0.5.5" -mime-db@~1.29.0: - version "1.29.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.29.0.tgz#48d26d235589651704ac5916ca06001914266878" +mime-db@~1.30.0: + version "1.30.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: - version "2.1.16" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.16.tgz#2b858a52e5ecd516db897ac2be87487830698e23" + version "2.1.17" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: - mime-db "~1.29.0" + mime-db "~1.30.0" -mime@1.3.4, mime@~1.3.4: +mime@1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1249,20 +1345,28 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +mime@~1.3.4: + version "1.3.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" + +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" -minimist@^1.2.0: +minimist@^1.2.0, minimist@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1313,17 +1417,17 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" nan@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" -nan@~2.5.0: - version "2.5.1" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" +nan@~2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" ncp@~2.0.0: version "2.0.0" @@ -1344,9 +1448,9 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36: - version "0.6.36" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" +node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.37: + version "0.6.37" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.37.tgz#3c872b236b2e266e4140578fe1ee88f693323a05" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1355,6 +1459,7 @@ node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36: request "^2.81.0" rimraf "^2.6.1" semver "^5.3.0" + tape "^4.6.3" tar "^2.2.1" tar-pack "^3.4.0" @@ -1409,6 +1514,14 @@ object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" +object-inspect@~1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.3.0.tgz#5b1eb8e6742e2ee83342a637034d844928ba2f6d" + +object-keys@^1.0.8: + version "1.0.11" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1475,8 +1588,8 @@ parse-json@^2.2.0: error-ex "^1.2.0" parseurl@~1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56" + version "1.3.2" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" path-exists@^2.0.0: version "2.1.0" @@ -1488,6 +1601,10 @@ path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" +path-parse@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -1516,16 +1633,15 @@ pg-pool@1.*: object-assign "4.1.0" pg-types@1.*: - version "1.12.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" + version "1.12.1" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2" dependencies: - ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: +"pg@github:cartodb/node-postgres#6.1.6-cdb1": version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1700,7 +1816,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1730,6 +1846,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1747,32 +1872,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.69.0, request@^2.81.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1799,6 +1899,31 @@ request@^2.69.0, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1811,6 +1936,18 @@ resolve@1.1.x: version "1.1.7" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" +resolve@~1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.4.0.tgz#a75be01c53da25d934a98ebd0e4c4a7312f92a86" + dependencies: + path-parse "^1.0.5" + +resumer@~0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/resumer/-/resumer-0.0.0.tgz#f1e8f461e4064ba39e82af3cdc2a8c893d076759" + dependencies: + through "~2.3.4" + right-align@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/right-align/-/right-align-0.1.3.tgz#61339b722fe6a3515689210d24e14c96148613ef" @@ -1818,8 +1955,8 @@ right-align@^0.1.1: align-text "^0.1.1" rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" + version "2.6.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" dependencies: glob "^7.0.5" @@ -1928,8 +2065,8 @@ source-map@^0.4.4: amdefine ">=0.0.4" source-map@^0.5.1, source-map@^0.5.6, source-map@~0.5.1: - version "0.5.6" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412" + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" source-map@~0.2.0: version "0.2.0" @@ -1955,10 +2092,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -1970,11 +2111,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.9" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.9.tgz#b2e7fbaa348380318d3834323918c3c351b8bf18" + version "3.1.11" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.11.tgz#84421b181b3e9f4939b7a3d712263d7946cfc96d" dependencies: - nan "~2.6.2" - node-pre-gyp "~0.6.36" + nan "~2.7.0" + node-pre-gyp "~0.6.37" srs@1.x: version "1.2.0" @@ -2026,6 +2167,14 @@ string-width@^1.0.1, string-width@^1.0.2: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" +string.prototype.trim@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz#d04de2c89e137f4d7d206f086b5ed2fae6be8cea" + dependencies: + define-properties "^1.1.2" + es-abstract "^1.5.0" + function-bind "^1.0.2" + string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" @@ -2076,6 +2225,24 @@ supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: dependencies: has-flag "^1.0.0" +tape@^4.6.3: + version "4.8.0" + resolved "https://registry.yarnpkg.com/tape/-/tape-4.8.0.tgz#f6a9fec41cc50a1de50fa33603ab580991f6068e" + dependencies: + deep-equal "~1.0.1" + defined "~1.0.0" + for-each "~0.3.2" + function-bind "~1.1.0" + glob "~7.1.2" + has "~1.0.1" + inherits "~2.0.3" + minimist "~1.2.0" + object-inspect "~1.3.0" + resolve "~1.4.0" + resumer "~0.0.0" + string.prototype.trim "~1.1.2" + through "~2.3.8" + tar-pack@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.0.tgz#23be2d7f671a8339376cbdb0b8fe3fdebf317984" @@ -2104,11 +2271,11 @@ through2@~0.2.3: readable-stream "~1.1.9" xtend "~2.1.1" -through@2: +through@2, through@~2.3.4, through@~2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2116,7 +2283,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2281,9 +2448,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.1.tgz#b5557fa6b0cfa13920904f57206b86f7aa054f74" +windshaft@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.2.tgz#72efe0dbc0d8d4bcba4211fdabd15dd2e0799df9" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From f133d983e85dfe9831cfb3d7d56653028f8c575f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Sep 2017 12:46:40 +0000 Subject: [PATCH 127/128] Release 3.12.10 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 0aad89fd..037467f8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.10 -Released 2017-mm-dd +Released 2017-09-18 - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). ## 3.12.9 From 33121871b0df4d90f8d6712301e6509f57b5618e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Sep 2017 12:47:37 +0000 Subject: [PATCH 128/128] Stubs next version --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 037467f8..2efa11b6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 3.12.11 +Released 2017-mm-dd + ## 3.12.10 Released 2017-09-18 - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). diff --git a/package.json b/package.json index 7e4f4b52..9c70ad72 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.10", + "version": "3.12.11", "description": "A map tile server for CartoDB", "keywords": [ "cartodb"