From 8694c120bc26885eceac73cb628dca8febf3df72 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 15 Mar 2017 11:00:10 +0100 Subject: [PATCH 001/402] Allow to overwrite layers filter in static maps images --- lib/cartodb/controllers/base.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 4ce9af17..b4ca8500 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -17,6 +17,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'zoom', 'lon', 'lat', + 'layer', // widgets & filters 'filters', // json 'own_filter', // 0, 1 From 8ca9c5bcf77fec999daefcd2f803163ecbb3eea2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 29 Mar 2017 15:56:30 +0200 Subject: [PATCH 002/402] Active GC interval Interval timer is configurable, disabling by using <=0 value. --- app.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app.js b/app.js index f6692d36..1220e8f6 100755 --- a/app.js +++ b/app.js @@ -136,3 +136,17 @@ process.on('SIGHUP', function() { process.on('uncaughtException', function(err) { global.logger.error('Uncaught exception: ' + err.stack); }); + +if (global.gc) { + var gcInterval = Number.isFinite(global.environment.gc_interval) ? + global.environment.gc_interval : + 10000; + + if (gcInterval > 0) { + setInterval(function gcForcedCycle() { + var start = Date.now(); + global.gc(); + global.statsClient.timing('windshaft.gc', Date.now() - start); + }, gcInterval); + } +} From d9968f2c9152c85bf8e321ed68afaa027a91c28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Mar 2017 16:02:37 +0200 Subject: [PATCH 003/402] Upgrade dependencies: grainstore@1.6.2 --- yarn.lock | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/yarn.lock b/yarn.lock index df4c739d..0690582e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -742,7 +742,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@3.2.3, "glob@~ 3.2.1": +glob@3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -781,6 +781,13 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" +"glob@~ 3.2.1": + version "3.2.11" + resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" + dependencies: + inherits "2" + minimatch "0.3" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -794,8 +801,8 @@ graceful-fs@~2.0.0: resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" grainstore@~1.6.0: - version "1.6.1" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.1.tgz#8950279ea737eb0ce403a85693642b4bed7f8e48" + version "1.6.2" + resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.2.tgz#f5d098a8da607f23db08f3263894c8f234a8a59e" dependencies: carto "0.16.3" debug "~2.2.0" @@ -984,9 +991,9 @@ isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" -isexe@^1.1.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/isexe/-/isexe-1.1.2.tgz#36f3e22e60750920f5e7241a476a8c6a42275ad0" +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" isstream@~0.1.2: version "0.1.2" @@ -1206,15 +1213,15 @@ millstone@0.6.17: underscore "~1.6.0" zipfile "~0.5.5" -mime-db@~1.26.0: - version "1.26.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.26.0.tgz#eaffcd0e4fc6935cf8134da246e2e6c35305adff" +mime-db@~1.27.0: + version "1.27.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1" mime-types@^2.1.12, mime-types@~2.1.13, mime-types@~2.1.6, mime-types@~2.1.7: - version "2.1.14" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee" + version "2.1.15" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed" dependencies: - mime-db "~1.26.0" + mime-db "~1.27.0" mime@1.3.4, mime@~1.3.4: version "1.3.4" @@ -1224,6 +1231,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" +minimatch@0.3: + version "0.3.0" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" + dependencies: + lru-cache "2" + sigmund "~1.0.0" + minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -2170,12 +2184,13 @@ type-is@~1.6.10, type-is@~1.6.6: mime-types "~2.1.13" uglify-js@^2.6: - version "2.8.14" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.14.tgz#25b15d1af39b21752ee33703adbf432e8bc8f77d" + version "2.8.18" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.18.tgz#925d14bae48ab62d1883b41afe6e2261662adb8e" dependencies: source-map "~0.5.1" - uglify-to-browserify "~1.0.0" yargs "~3.10.0" + optionalDependencies: + uglify-to-browserify "~1.0.0" uglify-to-browserify@~1.0.0: version "1.0.2" @@ -2235,10 +2250,10 @@ which-module@^1.0.0: resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" which@^1.1.1: - version "1.2.12" - resolved "https://registry.yarnpkg.com/which/-/which-1.2.12.tgz#de67b5e450269f194909ef23ece4ebe416fa1192" + version "1.2.14" + resolved "https://registry.yarnpkg.com/which/-/which-1.2.14.tgz#9a87c4378f03e827cecaf1acdf56c736c01c14e5" dependencies: - isexe "^1.1.1" + isexe "^2.0.0" wide-align@^1.1.0: version "1.1.0" From ac54179f14bebf7db83f3881e3ae2cd5bd4c0181 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 14:12:31 +0200 Subject: [PATCH 004/402] Update windshaft to 3.1.0 --- NEWS.md | 5 ++++- package.json | 4 ++-- yarn.lock | 28 +++++++--------------------- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index ade60a55..d7feec4d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,12 @@ # Changelog -## 3.1.2 +## 3.2.0 Released 2017-mm-dd +Announcements: + - Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0). + ## 3.1.1 Released 2017-03-23 diff --git a/package.json b/package.json index 14d6d2f8..356d96c7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.1.2", + "version": "3.2.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "3.0.1", + "windshaft": "3.1.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 0690582e..0c7cbe50 100644 --- a/yarn.lock +++ b/yarn.lock @@ -223,9 +223,9 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" @@ -742,7 +742,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@3.2.3: +glob@3.2.3, "glob@~ 3.2.1": version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -781,13 +781,6 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" -"glob@~ 3.2.1": - version "3.2.11" - resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" - dependencies: - inherits "2" - minimatch "0.3" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1231,13 +1224,6 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -minimatch@0.3: - version "0.3.0" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" - dependencies: - lru-cache "2" - sigmund "~1.0.0" - minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -1523,7 +1509,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2269,9 +2255,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.0.1.tgz#d06b4673704fe8f8f2e87c1f590c836659ab46f3" +windshaft@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.0.tgz#dfac2dd27a2db97e35231510743ad7db574da18f" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From f3fdd7ff257348c611970a05331b03a3df36d3da Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 14:32:48 +0200 Subject: [PATCH 005/402] Release 3.2.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index d7feec4d..0ed0ecd8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## 3.2.0 -Released 2017-mm-dd +Released 2017-03-30 Announcements: - Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0). From 5d74e1eafe4b2533fe4f3ecbba7d28650018e758 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 14:33:41 +0200 Subject: [PATCH 006/402] Stubs next version --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 0ed0ecd8..6e5ba7cb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 3.2.1 +Released 2017-mm-dd + ## 3.2.0 Released 2017-03-30 diff --git a/package.json b/package.json index 356d96c7..35d6095c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.2.0", + "version": "3.2.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 1ca56fb81c95ccee78232772119966167b8fc9f9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 17:52:25 +0200 Subject: [PATCH 007/402] Update news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 6e5ba7cb..08468e26 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ Released 2017-03-30 Announcements: - Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0). + - Active GC interval. ## 3.1.1 From 6468822295b8e0c141faf0467bd5d558401a2251 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 20:08:45 +0200 Subject: [PATCH 008/402] Remove layer param before creating a better solution --- lib/cartodb/controllers/base.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index b4ca8500..4ce9af17 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -17,7 +17,6 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'zoom', 'lon', 'lat', - 'layer', // widgets & filters 'filters', // json 'own_filter', // 0, 1 From ae5d82c41d58aa9c47bcb96fa88c072a84b1697b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 20:09:38 +0200 Subject: [PATCH 009/402] Add test to go red --- test/acceptance/named_maps_static_view.js | 43 +++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index 91026226..c7ddff2f 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -21,7 +21,7 @@ describe('named maps static view', function() { var IMAGE_TOLERANCE = 20; - function createTemplate(view) { + function createTemplate(view, layers) { return { version: '0.0.1', name: templateName, @@ -36,7 +36,7 @@ describe('named maps static view', function() { }, view: view, layergroup: { - layers: [ + layers: layers || [ { type: 'mapnik', options: { @@ -198,4 +198,43 @@ describe('named maps static view', function() { }); }); + it('should allow to select the layers to render', function (done) { + var view = { + bounds: { + west: 0, + south: 0, + east: 45, + north: 45 + } + }; + + var layers = [ + { + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer { marker-fill: <%= color %>; }', + cartocss_version: '2.3.0' + } + }, + { + type: 'mapnik', + options: { + sql: 'select ST_Transform(ST_MakeEnvelope(-45, -45, 45, 45, 4326), 3857) the_geom_webmercator', + cartocss: '#layer { polygon-fill: <%= color %>; }', + cartocss_version: '2.3.0' + } + } + ]; + templateMaps.addTemplate(username, createTemplate(view, layers), function (err) { + if (err) { + return done(err); + } + getStaticMap({ layer: 0 }, function(err, img) { + assert.ok(!err); + assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done); + }); + }); + }); + }); From 94299f04525b9fcc965de73f4199f55ded1ec309 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 20:12:55 +0200 Subject: [PATCH 010/402] Configure extra allowed params per endpoint via middleware Instead of making all params available in all endpoints, we control what endpoints allow what extra params. Dataviews endpoints should be migrated to this. --- lib/cartodb/controllers/base.js | 8 ++++++-- lib/cartodb/controllers/layergroup.js | 7 +++++-- lib/cartodb/controllers/named_maps.js | 3 ++- lib/cartodb/middleware/allow-query-params.js | 6 ++++++ 4 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 lib/cartodb/middleware/allow-query-params.js diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 4ce9af17..aa607a87 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -36,7 +36,7 @@ function BaseController(authApi, pgConnection) { module.exports = BaseController; -// jshint maxcomplexity:9 +// jshint maxcomplexity:10 /** * Whitelist input and get database name & default geometry type from * subdomain/user metadata held in CartoDB Redis @@ -77,7 +77,11 @@ BaseController.prototype.req2params = function(req, callback){ return; } - req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST); + var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST; + if (Array.isArray(req.context.allowedQueryParams)) { + allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams); + } + req.query = _.pick(req.query, allowedQueryParams); req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object var user = req.context.user; diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index f7215943..4119655f 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -6,6 +6,7 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); +var allowQueryParams = require('../middleware/allow-query-params'); var DataviewBackend = require('../backends/dataview'); var AnalysisStatusBackend = require('../backends/analysis-status'); @@ -67,11 +68,13 @@ LayergroupController.prototype.register = function(app) { this.attributes.bind(this)); app.get(app.base_url_mapconfig + - '/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware, + '/static/center/:token/:z/:lat/:lng/:width/:height.:format', + cors(), userMiddleware, allowQueryParams(['layer']), this.center.bind(this)); app.get(app.base_url_mapconfig + - '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware, + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', + cors(), userMiddleware, allowQueryParams(['layer']), this.bbox.bind(this)); // Undocumented/non-supported API endpoint methods. diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 4d22aeea..6681a893 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -8,6 +8,7 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); +var allowQueryParams = require('../middleware/allow-query-params'); function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { @@ -31,7 +32,7 @@ NamedMapsController.prototype.register = function(app) { this.tile.bind(this)); app.get(app.base_url_mapconfig + - '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, + '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']), this.staticMap.bind(this)); }; diff --git a/lib/cartodb/middleware/allow-query-params.js b/lib/cartodb/middleware/allow-query-params.js new file mode 100644 index 00000000..82c255f7 --- /dev/null +++ b/lib/cartodb/middleware/allow-query-params.js @@ -0,0 +1,6 @@ +module.exports = function allowQueryParams(params) { + return function allowQueryParamsMiddleware(req, res, next) { + req.context.allowedQueryParams = params; + next(); + }; +}; From cad02bfad78e1ad78863e2e5d593440cb4f8f34d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 30 Mar 2017 20:31:53 +0200 Subject: [PATCH 011/402] Remove all conditional branches to call req.profiler req.profiler is created in a middleware for all requests. --- lib/cartodb/api/auth_api.js | 8 ++---- lib/cartodb/controllers/base.js | 30 +++++++-------------- lib/cartodb/controllers/map.js | 16 +++-------- lib/cartodb/controllers/named_maps.js | 10 +++---- lib/cartodb/controllers/named_maps_admin.js | 12 +++------ test/unit/cartodb/ported/tile_stats.test.js | 4 +++ 6 files changed, 26 insertions(+), 54 deletions(-) diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js index 68533fe9..484d66b1 100644 --- a/lib/cartodb/api/auth_api.js +++ b/lib/cartodb/api/auth_api.js @@ -95,9 +95,7 @@ AuthApi.prototype.authorize = function(req, callback) { self.authorizedByAPIKey(user, req, this); }, function checkApiKey(err, authorized){ - if (req.profiler) { - req.profiler.done('authorizedByAPIKey'); - } + req.profiler.done('authorizedByAPIKey'); assert.ifError(err); // if not authorized by api_key, continue @@ -131,9 +129,7 @@ AuthApi.prototype.authorize = function(req, callback) { } self.pgConnection.setDBAuth(user, req.params, function(err) { - if (req.profiler) { - req.profiler.done('setDBAuth'); - } + req.profiler.done('setDBAuth'); callback(err, true); // authorized (or error) }); } diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 4ce9af17..90934b7d 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -61,9 +61,7 @@ BaseController.prototype.req2params = function(req, callback){ lzmaWorker.decompress( lzma, function(result) { - if (req.profiler) { - req.profiler.done('lzma'); - } + req.profiler.done('lzma'); try { delete req.query.lzma; _.extend(req.query, JSON.parse(result)); @@ -115,18 +113,14 @@ BaseController.prototype.req2params = function(req, callback){ // bring all query values onto req.params object _.extend(req.params, req.query); - if (req.profiler) { - req.profiler.done('req2params.setup'); - } + req.profiler.done('req2params.setup'); step( function getPrivacy(){ self.authApi.authorize(req, this); }, function validateAuthorization(err, authorized) { - if (req.profiler) { - req.profiler.done('authorize'); - } + req.profiler.done('authorize'); assert.ifError(err); if(!authorized) { err = new Error("Sorry, you are unauthorized (permission denied)"); @@ -167,9 +161,7 @@ BaseController.prototype.send = function(req, res, body, status, headers) { res.set('X-Served-By-DB-Host', req.params.dbhost); } - if (req.profiler) { - res.set('X-Tiler-Profiler', req.profiler.toJSONString()); - } + res.set('X-Tiler-Profiler', req.profiler.toJSONString()); if (headers) { res.set(headers); @@ -187,14 +179,12 @@ BaseController.prototype.send = function(req, res, body, status, headers) { res.send(body); } - if (req.profiler) { - try { - // May throw due to dns, see - // See http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - debug("error sending profiling stats: " + err); - } + try { + // May throw due to dns, see + // See http://github.com/CartoDB/Windshaft/issues/166 + req.profiler.sendStats(); + } catch (err) { + debug("error sending profiling stats: " + err); } }; // jshint maxcomplexity:6 diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 417cf266..cfd9e0d6 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -87,9 +87,7 @@ MapController.prototype.createPost = function(req, res) { }; MapController.prototype.instantiate = function(req, res) { - if (req.profiler) { - req.profiler.start('windshaft-cartodb.instance_template_post'); - } + req.profiler.start('windshaft-cartodb.instance_template_post'); this.instantiateTemplate(req, res, function prepareTemplateParams(callback) { if (!req.is('application/json')) { @@ -100,9 +98,7 @@ MapController.prototype.instantiate = function(req, res) { }; MapController.prototype.jsonp = function(req, res) { - if (req.profiler) { - req.profiler.start('windshaft-cartodb.instance_template_get'); - } + req.profiler.start('windshaft-cartodb.instance_template_get'); this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) { var err = null; @@ -303,9 +299,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la // take place before proceeding. Error will be logged // asynchronously this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) { - if (req.profiler) { - req.profiler.done('incMapviewCount'); - } + req.profiler.done('incMapviewCount'); if ( err ) { global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); } @@ -328,9 +322,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la QueryTables.getAffectedTablesFromQuery(connection, sql, this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { - if (req.profiler) { - req.profiler.done('queryTablesAndLastUpdated'); - } + req.profiler.done('queryTablesAndLastUpdated'); assert.ifError(err); // feed affected tables cache so it can be reused from, for instance, layergroup controller self.layergroupAffectedTables.set(dbName, layergroupId, result); diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 4d22aeea..f8f9b89c 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -100,9 +100,7 @@ NamedMapsController.prototype.tile = function(req, res) { self.tileBackend.getTile(namedMapProvider, req.params, this); }, function handleImage(err, tile, headers, stats) { - if (req.profiler) { - req.profiler.add(stats); - } + req.profiler.add(stats); if (err) { self.sendError(req, res, err, 'NAMED_MAP_TILE'); } else { @@ -176,10 +174,8 @@ NamedMapsController.prototype.staticMap = function(req, res) { }); }, function handleImage(err, image, headers, stats) { - if (req.profiler) { - req.profiler.done('render-' + format); - req.profiler.add(stats || {}); - } + req.profiler.done('render-' + format); + req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'STATIC_VIZ_MAP'); diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index 3d201049..08ad5322 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -91,9 +91,7 @@ NamedMapsAdminController.prototype.update = function(req, res) { NamedMapsAdminController.prototype.retrieve = function(req, res) { var self = this; - if (req.profiler) { - req.profiler.start('windshaft-cartodb.get_template'); - } + req.profiler.start('windshaft-cartodb.get_template'); var cdbuser = req.context.user; var tpl_id; @@ -127,9 +125,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) { NamedMapsAdminController.prototype.destroy = function(req, res) { var self = this; - if (req.profiler) { - req.profiler.start('windshaft-cartodb.delete_template'); - } + req.profiler.start('windshaft-cartodb.delete_template'); var cdbuser = req.context.user; var tpl_id; @@ -154,9 +150,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) { NamedMapsAdminController.prototype.list = function(req, res) { var self = this; - if ( req.profiler ) { - req.profiler.start('windshaft-cartodb.get_template_list'); - } + req.profiler.start('windshaft-cartodb.get_template_list'); var cdbuser = req.context.user; diff --git a/test/unit/cartodb/ported/tile_stats.test.js b/test/unit/cartodb/ported/tile_stats.test.js index e5ad622d..45b1fa29 100644 --- a/test/unit/cartodb/ported/tile_stats.test.js +++ b/test/unit/cartodb/ported/tile_stats.test.js @@ -26,12 +26,14 @@ describe('tile stats', function() { var layergroupController = new LayergroupController(); var reqMock = { + profiler: { toJSONString:function() {} }, params: { format: invalidFormat } }; var resMock = { status: function() { return this; }, + set: function() {}, json: function() {}, jsonp: function() {}, send: function() {} @@ -54,12 +56,14 @@ describe('tile stats', function() { } }); var reqMock = { + profiler: { toJSONString:function() {} }, params: { format: validFormat } }; var resMock = { status: function() { return this; }, + set: function() {}, json: function() {}, jsonp: function() {}, send: function() {} From ededc73fd74777d0d44b3d1613414fe3787a6c9f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 31 Mar 2017 18:39:29 +0200 Subject: [PATCH 012/402] Throw on invalid params argument --- lib/cartodb/middleware/allow-query-params.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cartodb/middleware/allow-query-params.js b/lib/cartodb/middleware/allow-query-params.js index 82c255f7..04a27033 100644 --- a/lib/cartodb/middleware/allow-query-params.js +++ b/lib/cartodb/middleware/allow-query-params.js @@ -1,4 +1,7 @@ module.exports = function allowQueryParams(params) { + if (!Array.isArray(params)) { + throw new Error('allowQueryParams must receive an Array of params'); + } return function allowQueryParamsMiddleware(req, res, next) { req.context.allowedQueryParams = params; next(); From 06427dc00963c6fbf76d5f76501605c55bc428e3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 3 Apr 2017 13:06:37 +0200 Subject: [PATCH 013/402] Bump version and add news --- NEWS.md | 5 ++++- package.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 08468e26..875a34ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,11 @@ # Changelog -## 3.2.1 +## 3.3.0 Released 2017-mm-dd +New features: + - Static map endpoints allow specifying the layers to render #653. + ## 3.2.0 Released 2017-03-30 diff --git a/package.json b/package.json index 35d6095c..de65a045 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.2.1", + "version": "3.3.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 508d495a2308eaaac65246734b3ec5854476d435 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 3 Apr 2017 13:07:05 +0200 Subject: [PATCH 014/402] Release 3.3.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 875a34ac..66f43117 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.3.0 -Released 2017-mm-dd +Released 2017-04-03 New features: - Static map endpoints allow specifying the layers to render #653. From 25669bb3f25dcf27213e55e48c675e71178e8f9b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 3 Apr 2017 13:07:53 +0200 Subject: [PATCH 015/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 66f43117..b8e57d37 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.3.1 +Released 2017-mm-dd + + ## 3.3.0 Released 2017-04-03 diff --git a/package.json b/package.json index de65a045..dc129692 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.3.0", + "version": "3.3.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 6bbda3d41ecc783530be913d99099b6204ff5b90 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 3 Apr 2017 15:52:39 +0200 Subject: [PATCH 016/402] Upgrade camshaft to 0.51.0 --- NEWS.md | 5 ++++- package.json | 4 ++-- yarn.lock | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/NEWS.md b/NEWS.md index b8e57d37..dada6dde 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,11 @@ # Changelog -## 3.3.1 +## 3.4.0 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.51.0](https://github.com/CartoDB/camshaft/releases/tag/0.51.0). + ## 3.3.0 Released 2017-04-03 diff --git a/package.json b/package.json index dc129692..16799d7b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.3.1", + "version": "3.4.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.50.3", + "camshaft": "0.51.0", "cartodb-psql": "~0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 0c7cbe50..29bb818b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.50.3: - version "0.50.3" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.50.3.tgz#dc6c5e3b0b39bc970c8ab6e66a61d905954eb85a" +camshaft@0.51.0: + version "0.51.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.51.0.tgz#f9bdf25a3971f57284deb10d83302487d4aa5e53" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -205,7 +205,7 @@ camshaft@0.50.3: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1730,7 +1730,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1755,7 +1755,7 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.81.0: +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -2084,7 +2084,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb1": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: version "2.3.1-cdb1" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" dependencies: @@ -2092,7 +2092,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb1": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb1: version "0.6.18-cdb1" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" dependencies: From 9c9609eb2bfa2adb9edc98f79f560f0e8cdb68e5 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 3 Apr 2017 16:21:29 +0200 Subject: [PATCH 017/402] Release 3.4.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index dada6dde..bce2ff84 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.4.0 -Released 2017-mm-dd +Released 2017-04-03 Announcements: - Upgrades camshaft to [0.51.0](https://github.com/CartoDB/camshaft/releases/tag/0.51.0). From 7a5aa7ba359d5890d83a6e517d10b1b23d3c46d3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 3 Apr 2017 16:22:15 +0200 Subject: [PATCH 018/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index bce2ff84..aaf9a2e4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.4.1 +Released 2017-mm-dd + + ## 3.4.0 Released 2017-04-03 diff --git a/package.json b/package.json index 16799d7b..db0703ad 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.4.0", + "version": "3.4.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 6d3644f13ba2e3f3a51ab8b491a96dcafd577a45 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 7 Apr 2017 13:29:30 +0200 Subject: [PATCH 019/402] Upgrades camshaft to 0.52.0 --- NEWS.md | 5 ++++- package.json | 4 ++-- yarn.lock | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index aaf9a2e4..bf4229f1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,11 @@ # Changelog -## 3.4.1 +## 3.5.0 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.52.0](https://github.com/CartoDB/camshaft/releases/tag/0.52.0). + ## 3.4.0 Released 2017-04-03 diff --git a/package.json b/package.json index db0703ad..6c665702 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.4.1", + "version": "3.5.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.51.0", + "camshaft": "0.52.0", "cartodb-psql": "~0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 29bb818b..8c739b00 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.51.0: - version "0.51.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.51.0.tgz#f9bdf25a3971f57284deb10d83302487d4aa5e53" +camshaft@0.52.0: + version "0.52.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.52.0.tgz#d7cac00884783c059fcaaa791879429b80c0e7b8" dependencies: async "^1.5.2" bunyan "1.8.1" From 5e4d1d5c1c6d0fbe39608d9a2b60819f556cf9da Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 10 Mar 2017 11:40:48 +0100 Subject: [PATCH 020/402] Get affected tables and add it to the layergroup --- .../adapter/analysis-mapconfig-adapter.js | 9 ++++++ test/acceptance/analysis/analysis-layers.js | 30 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js index b59684c7..09467ea9 100644 --- a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js @@ -115,6 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi } layer.options.sql = analysisSql; layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews)); + layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode); } else { missingNodesErrors.push( new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex) @@ -330,4 +331,12 @@ function AnalysisError(message) { this.message = message; } +function getAllAffectedTablesFromSourceNodes(node) { + var affectedTables = []; + var affectedTables = node.getAllInputNodes(function (node) { + return node.getType() === 'source'; + }).reduce(function(list, node) { return list.concat(node.getAffectedTables()); },[]); + return affectedTables; +} + require('util').inherits(AnalysisError, Error); diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index a745fb97..3a54ea2c 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -146,6 +146,36 @@ describe('analysis-layers', function() { }); }); + it('should have empty affected tables if it has only "source" node', function(done) { + var useCase = useCases[0]; + + var testClient = new TestClient(useCase.mapConfig, 1234); + + testClient.getLayergroup(function(err, layergroupResult) { + assert.ok(!err, err); + + var affected_tables = layergroupResult.metadata.layers[0].meta.affected_tables; + assert.equal(affected_tables.length, 0); + + testClient.drain(done); + }); + }); + + it('should have empty affected tables if it has a node other than "source"', function(done) { + var useCase = useCases[1]; + + var testClient = new TestClient(useCase.mapConfig, 1234); + + testClient.getLayergroup(function(err, layergroupResult) { + assert.ok(!err, err); + + var affected_tables = layergroupResult.metadata.layers[0].meta.affected_tables; + assert.equal(affected_tables[0], 'public.populated_places_simple_reduced'); + + testClient.drain(done); + }); + }); + it('should NOT fail for non-authenticated requests when it is just source', function(done) { var useCase = useCases[0]; From 0c387cf6d98882ddba7e4b3595d56ad8ef455c4d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 10 Mar 2017 17:22:07 +0100 Subject: [PATCH 021/402] Add more tests for x-cache-channel but with analysis --- test/acceptance/x_cache_channel.js | 247 ++++++++++++++++++----------- 1 file changed, 152 insertions(+), 95 deletions(-) diff --git a/test/acceptance/x_cache_channel.js b/test/acceptance/x_cache_channel.js index 4ebb45e8..cd12e6ce 100644 --- a/test/acceptance/x_cache_channel.js +++ b/test/acceptance/x_cache_channel.js @@ -25,32 +25,92 @@ describe('get requests x-cache-channel', function() { status: 200 }; - var mapConfig = { - version: '1.3.0', - layers: [ + var mapConfigs = [ + { + "description": "header should be present", + "data": + { + version: '1.4.0', + layers: [ + { + options: { + source: { + id: "2570e105-7b37-40d2-bdf4-1af889598745" + }, + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; }', + cartocss_version: '2.3.0', + attributes: { + id:'cartodb_id', + columns: [ + 'name', + 'address' + ] + } + } + } + ], + analyses: [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from test_table limit 2" + } + } + ] + }, + }, + { + "description": "header should be present and be composed with source table name", + "data": { - options: { - sql: 'select * from test_table limit 2', - cartocss: '#layer { marker-fill:red; }', - cartocss_version: '2.3.0', - attributes: { - id:'cartodb_id', - columns: [ - 'name', - 'address' - ] + version: '1.5.0', + layers: [ + { + options: { + source: { + id: "2570e105-7b37-40d2-bdf4-1af889598745" + }, + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; }', + cartocss_version: '2.3.0', + attributes: { + id:'cartodb_id', + columns: [ + 'name', + 'address' + ] + } + } } - } + ], + analyses: [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from test_table limit 2" + } + }, + "radius": 50000 + } + } + ] } - ] - }; + }]; - var layergroupRequest = { - url: '/api/v1/map?config=' + encodeURIComponent(JSON.stringify(mapConfig)), - method: 'GET', - headers: { - host: 'localhost' - } + var layergroupRequest = function(mapConfig) { + return { + url: '/api/v1/map?api_key=1234&config=' + encodeURIComponent(JSON.stringify(mapConfig)), + method: 'GET', + headers: { + host: 'localhost' + } + }; }; function getRequest(url, addApiKey, callbackName) { @@ -101,10 +161,10 @@ describe('get requests x-cache-channel', function() { }; } - function withLayergroupId(callback) { + function withLayergroupId(mapConfig, callback) { assert.response( server, - layergroupRequest, + layergroupRequest(mapConfig), statusOkResponse, function(res, err) { if (err) { @@ -118,75 +178,76 @@ describe('get requests x-cache-channel', function() { ); } - describe('header should be present', function() { + mapConfigs.forEach(function(mapConfigData) { + describe(mapConfigData.description, function() { + var mapConfig = mapConfigData.data; + it('/api/v1/map Map instantiation', function(done) { + var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table'); + withLayergroupId(mapConfig, function(err, layergroupId, res) { + testFn(res); + }); + }); - it('/api/v1/map Map instantiation', function(done) { - var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table'); - withLayergroupId(function(err, layergroupId, res) { - testFn(res); + it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'), + validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + ); + }); + }); + + it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'), + validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + ); + }); + }); + + it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'), + validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + ); + }); + }); + + it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'), + validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + ); + }); + }); + + it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'), + validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + ); + }); + }); + + it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'), + validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + ); + }); }); }); - - it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - }); describe('header should NOT be present', function() { @@ -238,7 +299,7 @@ describe('get requests x-cache-channel', function() { auth: { method: 'open' }, - layergroup: mapConfig + layergroup: mapConfigs[0].data }; var namedMapRequest = { @@ -298,10 +359,6 @@ describe('get requests x-cache-channel', function() { noXCacheChannelHeader(done) ); }); - }); - }); - - }); From fa6493ae444de4d30527f187a544c62b52458cdd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 13 Mar 2017 17:28:29 +0100 Subject: [PATCH 022/402] Affected tables are now included in X-Cache-Channel --- lib/cartodb/controllers/layergroup.js | 16 +++++----- lib/cartodb/controllers/map.js | 14 ++++++--- .../adapter/analysis-mapconfig-adapter.js | 1 - test/acceptance/analysis/analysis-layers.js | 30 ------------------ test/acceptance/templates.js | 6 ++-- test/acceptance/x_cache_channel.js | 31 +++++++++++-------- 6 files changed, 41 insertions(+), 57 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 4119655f..10b5324b 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -391,13 +391,15 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg function getSQL(err, mapConfig) { assert.ifError(err); - var queries = mapConfig.getLayers() - .map(function(lyr) { - return lyr.options.sql; - }) - .filter(function(sql) { - return !!sql; - }); + var queries = []; + mapConfig.getLayers().map(function(layer) { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(function(table) { + queries.push('SELECT * FROM ' + table + ' LIMIT 0'); + }); + } + }); return queries.length ? queries.join(';') : null; }, diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index cfd9e0d6..3b06ac02 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -306,9 +306,15 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la done(); }); - var sql = mapconfig.getLayers().map(function(layer) { - return layer.options.sql; - }).join(';'); + var sql = []; + mapconfig.getLayers().map(function(layer) { + sql.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(function(table) { + sql.push('SELECT * FROM ' + table + ' LIMIT 0'); + }); + } + }); var dbName = req.params.dbname; var layergroupId = layergroup.layergroupid; @@ -319,7 +325,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la }, function getAffectedTablesAndLastUpdatedTime(err, connection) { assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); + QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { req.profiler.done('queryTablesAndLastUpdated'); diff --git a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js index 09467ea9..c6467488 100644 --- a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js @@ -332,7 +332,6 @@ function AnalysisError(message) { } function getAllAffectedTablesFromSourceNodes(node) { - var affectedTables = []; var affectedTables = node.getAllInputNodes(function (node) { return node.getType() === 'source'; }).reduce(function(list, node) { return list.concat(node.getAffectedTables()); },[]); diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index 3a54ea2c..a745fb97 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -146,36 +146,6 @@ describe('analysis-layers', function() { }); }); - it('should have empty affected tables if it has only "source" node', function(done) { - var useCase = useCases[0]; - - var testClient = new TestClient(useCase.mapConfig, 1234); - - testClient.getLayergroup(function(err, layergroupResult) { - assert.ok(!err, err); - - var affected_tables = layergroupResult.metadata.layers[0].meta.affected_tables; - assert.equal(affected_tables.length, 0); - - testClient.drain(done); - }); - }); - - it('should have empty affected tables if it has a node other than "source"', function(done) { - var useCase = useCases[1]; - - var testClient = new TestClient(useCase.mapConfig, 1234); - - testClient.getLayergroup(function(err, layergroupResult) { - assert.ok(!err, err); - - var affected_tables = layergroupResult.metadata.layers[0].meta.affected_tables; - assert.equal(affected_tables[0], 'public.populated_places_simple_reduced'); - - testClient.drain(done); - }); - }); - it('should NOT fail for non-authenticated requests when it is just source', function(done) { var useCase = useCases[0]; diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 7201b7fb..74d34d82 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -1052,8 +1052,9 @@ describe('template_api', function() { 'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); var cc = res.headers['x-cache-channel']; + var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1'; assert.ok(cc); - assert.ok(cc.match, /ciao/, cc); + assert.equal(cc, expectedCC); // hack simulating restart... server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache var get_request = { @@ -1072,8 +1073,9 @@ describe('template_api', function() { 'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); var cc = res.headers['x-cache-channel']; + var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1'; assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart"); - assert.ok(cc.match, /ciao/, cc); + assert.equal(cc, expectedCC); return null; }, function deleteTemplate(err) diff --git a/test/acceptance/x_cache_channel.js b/test/acceptance/x_cache_channel.js index cd12e6ce..298c0ef0 100644 --- a/test/acceptance/x_cache_channel.js +++ b/test/acceptance/x_cache_channel.js @@ -28,6 +28,7 @@ describe('get requests x-cache-channel', function() { var mapConfigs = [ { "description": "header should be present", + "x_cache_channel": "test_windshaft_cartodb_user_1_db:public.test_table", "data": { version: '1.4.0', @@ -63,6 +64,9 @@ describe('get requests x-cache-channel', function() { }, { "description": "header should be present and be composed with source table name", + "x_cache_channel": "test_windshaft_cartodb_user_1_db:" + + "public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d" + + ",public.test_table", "data": { version: '1.5.0', @@ -181,8 +185,9 @@ describe('get requests x-cache-channel', function() { mapConfigs.forEach(function(mapConfigData) { describe(mapConfigData.description, function() { var mapConfig = mapConfigData.data; + var expectedCacheChannel = mapConfigData.x_cache_channel; it('/api/v1/map Map instantiation', function(done) { - var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table'); + var testFn = validateXCacheChannel(done, expectedCacheChannel); withLayergroupId(mapConfig, function(err, layergroupId, res) { testFn(res); }); @@ -192,8 +197,8 @@ describe('get requests x-cache-channel', function() { withLayergroupId(mapConfig, function(err, layergroupId) { assert.response( server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true), + validateXCacheChannel(done, expectedCacheChannel) ); }); }); @@ -202,8 +207,8 @@ describe('get requests x-cache-channel', function() { withLayergroupId(mapConfig, function(err, layergroupId) { assert.response( server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true), + validateXCacheChannel(done, expectedCacheChannel) ); }); }); @@ -212,8 +217,8 @@ describe('get requests x-cache-channel', function() { withLayergroupId(mapConfig, function(err, layergroupId) { assert.response( server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true), + validateXCacheChannel(done, expectedCacheChannel) ); }); }); @@ -222,8 +227,8 @@ describe('get requests x-cache-channel', function() { withLayergroupId(mapConfig, function(err, layergroupId) { assert.response( server, - getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true), + validateXCacheChannel(done, expectedCacheChannel) ); }); }); @@ -232,8 +237,8 @@ describe('get requests x-cache-channel', function() { withLayergroupId(mapConfig, function(err, layergroupId) { assert.response( server, - getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true), + validateXCacheChannel(done, expectedCacheChannel) ); }); }); @@ -242,8 +247,8 @@ describe('get requests x-cache-channel', function() { withLayergroupId(mapConfig, function(err, layergroupId) { assert.response( server, - getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') + getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true), + validateXCacheChannel(done, expectedCacheChannel) ); }); }); From 9707881bf90dfaf94cfade2018aa2708a128e0cd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 13 Mar 2017 18:36:13 +0100 Subject: [PATCH 023/402] Include check for surrogate-key header and renamed the test file --- lib/cartodb/controllers/layergroup.js | 2 +- lib/cartodb/controllers/map.js | 2 +- .../adapter/analysis-mapconfig-adapter.js | 4 +- .../cache_headers.js} | 76 +++++++++++-------- 4 files changed, 49 insertions(+), 35 deletions(-) rename test/acceptance/{x_cache_channel.js => cache/cache_headers.js} (81%) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 10b5324b..79268d6e 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -392,7 +392,7 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg assert.ifError(err); var queries = []; - mapConfig.getLayers().map(function(layer) { + mapConfig.getLayers().forEach(function(layer) { queries.push(layer.options.sql); if (layer.options.affected_tables) { layer.options.affected_tables.map(function(table) { diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 3b06ac02..b3ca2b4a 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -307,7 +307,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la }); var sql = []; - mapconfig.getLayers().map(function(layer) { + mapconfig.getLayers().forEach(function(layer) { sql.push(layer.options.sql); if (layer.options.affected_tables) { layer.options.affected_tables.map(function(table) { diff --git a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js index c6467488..8cb63f48 100644 --- a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js @@ -334,7 +334,9 @@ function AnalysisError(message) { function getAllAffectedTablesFromSourceNodes(node) { var affectedTables = node.getAllInputNodes(function (node) { return node.getType() === 'source'; - }).reduce(function(list, node) { return list.concat(node.getAffectedTables()); },[]); + }).reduce(function(list, node) { + return list.concat(node.getAffectedTables()); + },[]); return affectedTables; } diff --git a/test/acceptance/x_cache_channel.js b/test/acceptance/cache/cache_headers.js similarity index 81% rename from test/acceptance/x_cache_channel.js rename to test/acceptance/cache/cache_headers.js index 298c0ef0..13ca77dc 100644 --- a/test/acceptance/x_cache_channel.js +++ b/test/acceptance/cache/cache_headers.js @@ -1,16 +1,16 @@ -var testHelper = require('../support/test_helper'); +var testHelper = require('../../support/test_helper'); -var assert = require('../support/assert'); +var assert = require('../../support/assert'); var qs = require('querystring'); -var CartodbWindshaft = require('../../lib/cartodb/server'); -var serverOptions = require('../../lib/cartodb/server_options'); +var CartodbWindshaft = require('../../../lib/cartodb/server'); +var serverOptions = require('../../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -var LayergroupToken = require('../support/layergroup-token'); +var LayergroupToken = require('../../support/layergroup-token'); -describe('get requests x-cache-channel', function() { +describe('get requests with cache headers', function() { var keysToDelete; beforeEach(function() { @@ -27,11 +27,14 @@ describe('get requests x-cache-channel', function() { var mapConfigs = [ { - "description": "header should be present", - "x_cache_channel": "test_windshaft_cartodb_user_1_db:public.test_table", + "description": "cache headers should be present", + "cache_headers": { + "x_cache_channel": "test_windshaft_cartodb_user_1_db:public.test_table", + "surrogate_keys": "t:77pJnX" + }, "data": { - version: '1.4.0', + version: '1.5.0', layers: [ { options: { @@ -63,10 +66,13 @@ describe('get requests x-cache-channel', function() { }, }, { - "description": "header should be present and be composed with source table name", - "x_cache_channel": "test_windshaft_cartodb_user_1_db:" + - "public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d" + - ",public.test_table", + "description": "cache headers should be present and be composed with source table name", + "cache_headers": { + "x_cache_channel": "test_windshaft_cartodb_user_1_db:" + + "public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d" + + ",public.test_table", + "surrogate_keys": "t:77pJnX t:iL4eth" + }, "data": { version: '1.5.0', @@ -136,22 +142,24 @@ describe('get requests x-cache-channel', function() { }; } - function validateXCacheChannel(done, expectedCacheChannel) { + function validateCacheHeaders(done, expectedCacheHeaders) { return function(res, err) { if (err) { return done(err); } assert.ok(res.headers['x-cache-channel']); - if (expectedCacheChannel) { - assert.equal(res.headers['x-cache-channel'], expectedCacheChannel); + assert.ok(res.headers['surrogate-key']); + if (expectedCacheHeaders) { + assert.equal(res.headers['x-cache-channel'], expectedCacheHeaders.x_cache_channel); + assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys); } done(); }; } - function noXCacheChannelHeader(done) { + function noCacheHeaders(done) { return function(res, err) { if (err) { return done(err); @@ -161,6 +169,10 @@ describe('get requests x-cache-channel', function() { !res.headers['x-cache-channel'], 'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`' ); + assert.ok( + !res.headers['surrogate-key'], + 'did not expect surrogate-key header, got: `' + res.headers['surrogate-key'] + '`' + ); done(); }; } @@ -185,9 +197,9 @@ describe('get requests x-cache-channel', function() { mapConfigs.forEach(function(mapConfigData) { describe(mapConfigData.description, function() { var mapConfig = mapConfigData.data; - var expectedCacheChannel = mapConfigData.x_cache_channel; + var expectedCacheHeaders = mapConfigData.cache_headers; it('/api/v1/map Map instantiation', function(done) { - var testFn = validateXCacheChannel(done, expectedCacheChannel); + var testFn = validateCacheHeaders(done, expectedCacheHeaders); withLayergroupId(mapConfig, function(err, layergroupId, res) { testFn(res); }); @@ -198,7 +210,7 @@ describe('get requests x-cache-channel', function() { assert.response( server, getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true), - validateXCacheChannel(done, expectedCacheChannel) + validateCacheHeaders(done, expectedCacheHeaders) ); }); }); @@ -208,7 +220,7 @@ describe('get requests x-cache-channel', function() { assert.response( server, getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true), - validateXCacheChannel(done, expectedCacheChannel) + validateCacheHeaders(done, expectedCacheHeaders) ); }); }); @@ -218,7 +230,7 @@ describe('get requests x-cache-channel', function() { assert.response( server, getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true), - validateXCacheChannel(done, expectedCacheChannel) + validateCacheHeaders(done, expectedCacheHeaders) ); }); }); @@ -228,7 +240,7 @@ describe('get requests x-cache-channel', function() { assert.response( server, getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true), - validateXCacheChannel(done, expectedCacheChannel) + validateCacheHeaders(done, expectedCacheHeaders) ); }); }); @@ -238,7 +250,7 @@ describe('get requests x-cache-channel', function() { assert.response( server, getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true), - validateXCacheChannel(done, expectedCacheChannel) + validateCacheHeaders(done, expectedCacheHeaders) ); }); }); @@ -248,21 +260,21 @@ describe('get requests x-cache-channel', function() { assert.response( server, getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true), - validateXCacheChannel(done, expectedCacheChannel) + validateCacheHeaders(done, expectedCacheHeaders) ); }); }); }); }); - describe('header should NOT be present', function() { + describe('cache headers should NOT be present', function() { it('/', function(done) { assert.response( server, getRequest('/'), statusOkResponse, - noXCacheChannelHeader(done) + noCacheHeaders(done) ); }); @@ -271,7 +283,7 @@ describe('get requests x-cache-channel', function() { server, getRequest('/version'), statusOkResponse, - noXCacheChannelHeader(done) + noCacheHeaders(done) ); }); @@ -280,7 +292,7 @@ describe('get requests x-cache-channel', function() { server, getRequest('/health'), statusOkResponse, - noXCacheChannelHeader(done) + noCacheHeaders(done) ); }); @@ -289,7 +301,7 @@ describe('get requests x-cache-channel', function() { server, getRequest('/api/v1/map/named', true), statusOkResponse, - noXCacheChannelHeader(done) + noCacheHeaders(done) ); }); @@ -352,7 +364,7 @@ describe('get requests x-cache-channel', function() { server, getRequest('/api/v1/map/named/' + templateName, true), statusOkResponse, - noXCacheChannelHeader(done) + noCacheHeaders(done) ); }); @@ -361,7 +373,7 @@ describe('get requests x-cache-channel', function() { server, getRequest('/api/v1/map/named/' + templateName, true, 'cb'), statusOkResponse, - noXCacheChannelHeader(done) + noCacheHeaders(done) ); }); }); From 4132bc755da715490538336bf72c0aad82a153ec Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 14 Mar 2017 14:06:50 +0100 Subject: [PATCH 024/402] Add cdb_invalidate_varnish function fixture to tests --- test/support/prepare_db.sh | 2 +- test/support/sql/cdb_invalidate_varnish.sql | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 test/support/sql/cdb_invalidate_varnish.sql diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 1a6d929c..4fbf4b7a 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -75,7 +75,7 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" - LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check' + LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' CURL_ARGS="" diff --git a/test/support/sql/cdb_invalidate_varnish.sql b/test/support/sql/cdb_invalidate_varnish.sql new file mode 100644 index 00000000..7cd2d8f1 --- /dev/null +++ b/test/support/sql/cdb_invalidate_varnish.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE FUNCTION CDB_Invalidate_Varnish(table_name TEXT) +RETURNS void AS +$$ +BEGIN +END; +$$ LANGUAGE PLPGSQL; \ No newline at end of file From 125c39967c30234f1ead15d2e33c5f82532e315a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 14 Mar 2017 14:32:36 +0100 Subject: [PATCH 025/402] Make the cache headers tests idempotent --- test/acceptance/cache/cache_headers.js | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/test/acceptance/cache/cache_headers.js b/test/acceptance/cache/cache_headers.js index 13ca77dc..2cd916af 100644 --- a/test/acceptance/cache/cache_headers.js +++ b/test/acceptance/cache/cache_headers.js @@ -29,7 +29,10 @@ describe('get requests with cache headers', function() { { "description": "cache headers should be present", "cache_headers": { - "x_cache_channel": "test_windshaft_cartodb_user_1_db:public.test_table", + "x_cache_channel": { + "db_name": "test_windshaft_cartodb_user_1_db", + "tables": ["public.test_table"] + }, "surrogate_keys": "t:77pJnX" }, "data": @@ -68,9 +71,11 @@ describe('get requests with cache headers', function() { { "description": "cache headers should be present and be composed with source table name", "cache_headers": { - "x_cache_channel": "test_windshaft_cartodb_user_1_db:" + - "public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d" + - ",public.test_table", + "x_cache_channel": { + "db_name": "test_windshaft_cartodb_user_1_db", + "tables": ["public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d", + "public.test_table"] + }, "surrogate_keys": "t:77pJnX t:iL4eth" }, "data": @@ -151,7 +156,7 @@ describe('get requests with cache headers', function() { assert.ok(res.headers['x-cache-channel']); assert.ok(res.headers['surrogate-key']); if (expectedCacheHeaders) { - assert.equal(res.headers['x-cache-channel'], expectedCacheHeaders.x_cache_channel); + validateXChannelHeaders(res.headers, expectedCacheHeaders); assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys); } @@ -159,6 +164,13 @@ describe('get requests with cache headers', function() { }; } + function validateXChannelHeaders(headers, expectedCacheHeaders) { + var dbName = headers['x-cache-channel'].split(':')[0]; + var tables = headers['x-cache-channel'].split(':')[1].split(',').sort(); + assert.equal(dbName, expectedCacheHeaders.x_cache_channel.db_name); + assert.deepEqual(tables, expectedCacheHeaders.x_cache_channel.tables.sort()); + } + function noCacheHeaders(done) { return function(res, err) { if (err) { From e8a0f6b7b672ae3947b2e5f7d6f979492dd0c2e0 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 14 Mar 2017 12:57:46 +0100 Subject: [PATCH 026/402] Point to camshaft branch to test properly --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6c665702..53c38c3f 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.52.0", + "camshaft": "cartodb/camshaft#add_affected_tables", "cartodb-psql": "~0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", From 92ec17218ba9713000cfa64c762a823d988511ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 10 Apr 2017 11:36:17 +0200 Subject: [PATCH 027/402] Upgrade camshaft to 0.53.0 --- package.json | 2 +- yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 53c38c3f..be83898e 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "cartodb/camshaft#add_affected_tables", + "camshaft": "0.53.0", "cartodb-psql": "~0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 8c739b00..92c3c673 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.52.0: - version "0.52.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.52.0.tgz#d7cac00884783c059fcaaa791879429b80c0e7b8" +camshaft@0.53.0: + version "0.53.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.53.0.tgz#89708ca89bea93c4d94b0587b692935e23f1e9d6" dependencies: async "^1.5.2" bunyan "1.8.1" From 891bc818b26115d35dfd40905b3af9b44f7aaca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 10 Apr 2017 12:20:44 +0200 Subject: [PATCH 028/402] Release 3.5.0 --- NEWS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index bf4229f1..94897c90 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,13 @@ # Changelog ## 3.5.0 -Released 2017-mm-dd +Released 2017-04-10 + +Bug fixes: + - Fix invalidation of cache for maps with analyses #638. Announcements: - - Upgrades camshaft to [0.52.0](https://github.com/CartoDB/camshaft/releases/tag/0.52.0). + - Upgrades camshaft to [0.53.0](https://github.com/CartoDB/camshaft/releases/tag/0.53.0). ## 3.4.0 From c6a62cee616352b43493ddd888855c1d3a3dae64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 10 Apr 2017 16:13:30 +0200 Subject: [PATCH 029/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 94897c90..01487a6e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.5.1 +Released 2017-mm-dd + + ## 3.5.0 Released 2017-04-10 diff --git a/package.json b/package.json index be83898e..a43e469c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.5.0", + "version": "3.5.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From a4ed37bdfc0647afa0bd32cbba7a53dc0d95645d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Apr 2017 18:19:39 +0200 Subject: [PATCH 030/402] Upgrade camshaft to 0.53.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a43e469c..406b2c3b 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.53.0", + "camshaft": "0.53.1", "cartodb-psql": "~0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", From 227112c7aabedc1d8fcdc5e9f4412cbebe296db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Apr 2017 18:27:54 +0200 Subject: [PATCH 031/402] Update yarn.lock --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 92c3c673..83f96064 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.53.0: - version "0.53.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.53.0.tgz#89708ca89bea93c4d94b0587b692935e23f1e9d6" +camshaft@0.53.1: + version "0.53.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.53.1.tgz#ba280be034e161c4989e1c81bafcdeb42d10ef7b" dependencies: async "^1.5.2" bunyan "1.8.1" From a386abf5a572b487e5de14dded9c7b9c59b7507b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Apr 2017 18:34:18 +0200 Subject: [PATCH 032/402] Release 3.5.1 --- NEWS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 01487a6e..cb0d9470 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,10 @@ # Changelog ## 3.5.1 -Released 2017-mm-dd +Released 2017-04-11 + +Announcements: + - Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1). ## 3.5.0 From 2756252368faf3b54f53b3499827b7fea5e52c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Apr 2017 19:23:04 +0200 Subject: [PATCH 033/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index cb0d9470..9a18d346 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.5.2 +Released 2017-mm-dd + + ## 3.5.1 Released 2017-04-11 diff --git a/package.json b/package.json index 406b2c3b..4a3003ba 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.5.1", + "version": "3.5.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 02b9f85b16f520fee501324bd30eed4049b7fe79 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 20 Apr 2017 18:25:57 +0200 Subject: [PATCH 034/402] Upgrades camshaft to 0.54.0 --- NEWS.md | 5 ++++- package.json | 4 ++-- yarn.lock | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9a18d346..ccb90b60 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,11 @@ # Changelog -## 3.5.2 +## 3.6.0 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0). + ## 3.5.1 Released 2017-04-11 diff --git a/package.json b/package.json index 4a3003ba..9a7c5dd0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.5.2", + "version": "3.6.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.53.1", + "camshaft": "0.54.0", "cartodb-psql": "~0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 83f96064..9a3c48fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.53.1: - version "0.53.1" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.53.1.tgz#ba280be034e161c4989e1c81bafcdeb42d10ef7b" +camshaft@0.54.0: + version "0.54.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.0.tgz#9cb23b827dee0f54d0a39b8af3213692f11adf62" dependencies: async "^1.5.2" bunyan "1.8.1" From 781e5a71bfc4cb09961b88ed20f6234c7704a0fe Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 12:29:08 +0200 Subject: [PATCH 035/402] Upgrade camshaft to 0.54.1 --- NEWS.md | 7 +++++++ package.json | 4 ++-- yarn.lock | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index ccb90b60..c19611de 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,12 @@ # Changelog +## 3.6.1 +Released 2017-mm-dd + +Announcements: + - Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1). + + ## 3.6.0 Released 2017-mm-dd diff --git a/package.json b/package.json index 9a7c5dd0..693858ad 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.0", + "version": "3.6.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.54.0", + "camshaft": "0.54.1", "cartodb-psql": "~0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 9a3c48fe..1842800d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.54.0: - version "0.54.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.0.tgz#9cb23b827dee0f54d0a39b8af3213692f11adf62" +camshaft@0.54.1: + version "0.54.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.1.tgz#0eea25ee9e5a1cf32b0f59fef48f8f3590ebfe17" dependencies: async "^1.5.2" bunyan "1.8.1" From 47944671c66e90ea1c2aa0e04f72b97312f1925f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 14:43:23 +0200 Subject: [PATCH 036/402] fix release date --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index c19611de..bf364b10 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,7 +8,7 @@ Announcements: ## 3.6.0 -Released 2017-mm-dd +Released 2017-04-20 Announcements: - Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0). From 4b6181039d86bfffbe94d4dc7a9214bb043d7d6e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 14:43:46 +0200 Subject: [PATCH 037/402] Release 3.6.1 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index bf364b10..347ea22c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.6.1 -Released 2017-mm-dd +Released 2017-04-24 Announcements: - Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1). From 14d37268d610e5e8afbba3bc0fdef181bd7a8663 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 14:44:38 +0200 Subject: [PATCH 038/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 347ea22c..f1464b9d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.6.2 +Released 2017-mm-dd + + ## 3.6.1 Released 2017-04-24 diff --git a/package.json b/package.json index 693858ad..8706b7f0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.1", + "version": "3.6.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 883f87c7c8c9932604c80fdd862c7ca18d06169d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 18:28:51 +0200 Subject: [PATCH 039/402] Document gc_interval configuration entry --- config/environments/development.js.example | 3 +++ config/environments/production.js.example | 3 +++ config/environments/staging.js.example | 3 +++ config/environments/test.js.example | 3 +++ 4 files changed, 12 insertions(+) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 6800d4c6..9bc4157b 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.localhost' diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 41e2353d..b305deef 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 170e5e5f..fa3c7cb6 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 3c940ac6..465b1774 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '(.*)' From fed953d1950b3880bba458b2707b5e19985f225b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 24 Apr 2017 18:55:08 +0200 Subject: [PATCH 040/402] Support mvt tiles --- test/support/test-client.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 75fa51e0..5998912f 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -485,6 +485,13 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { expectedResponse.headers['Content-Type'] = 'image/png'; } + var isMvt = format.match(/mvt$/); + + if (isMvt) { + request.encoding = 'binary'; + expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; + } + assert.response(server, request, expectedResponse, function(res, err) { assert.ifError(err); @@ -492,7 +499,13 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { if (isPng) { obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } else { + } + else if (isMvt) { + obj = new mapnik.VectorTile(z, x, y); + obj.setDataSync(new Buffer(res.body, 'binary')); + } + + else { obj = JSON.parse(res.body); } From 06353941e6ef6200e7130e2ccf0389b10c098996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 24 Apr 2017 18:56:15 +0200 Subject: [PATCH 041/402] Implement test to exercise buffer-size configuration by format --- test/acceptance/buffer-size-format.js | 124 ++++++++++++++++++ .../tile-7.64.48-buffer-size-1.png | Bin 0 -> 1379 bytes .../tile-7.64.48-buffer-size-128.png | Bin 0 -> 2458 bytes 3 files changed, 124 insertions(+) create mode 100644 test/acceptance/buffer-size-format.js create mode 100644 test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png create mode 100644 test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js new file mode 100644 index 00000000..bbf643bc --- /dev/null +++ b/test/acceptance/buffer-size-format.js @@ -0,0 +1,124 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); +var IMAGE_TOLERANCE_PER_MIL = 20; +var mapnik = require('windshaft').mapnik; + +function createMapConfig (bufferSize) { + return { + version: '1.6.0', + buffersize: bufferSize, + layers: [{ + type: "cartodb", + options: { + sql: [ + 'select', + ' *', + 'from', + ' populated_places_simple_reduced', + ].join('\n'), + cartocss: [ + '#layer {', + ' polygon-fill: #374C70;', + ' polygon-opacity: 0.9;', + ' line-width: 1;', + ' line-color: #FFF;', + ' line-opacity: 0.5;', + '}', + '#layer::labels {', + ' text-name: [name];', + ' text-face-name: \'DejaVu Sans Book\';', + ' text-size: 20;', + ' text-fill: #FFFFFF;', + ' text-label-position-tolerance: 0;', + ' text-halo-radius: 1;', + ' text-halo-fill: #6F808D;', + ' text-dy: -10;', + ' text-allow-overlap: true;', + ' text-placement: point;', + ' text-placement-type: dummy;', + '}' + ].join('\n'), + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + }; +} + +describe.only('buffer size per format', function () { + var testCases = [ + { + desc: 'should get png tile using buffer-size 1', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png', + mapConfig: createMapConfig({ png: 1, 'grid.json': 1 }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + mapConfig: createMapConfig({ png: 128, 'grid.json': 128 }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get mvt tile using buffer-size 1', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png', + mapConfig: createMapConfig({ mvt: 1 }), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features; + assert.equal(features.length, 1); + + var map = new mapnik.Map(256, 256); + tile.render(map, new mapnik.Image(256, 256), function (err, image) { + assert.ifError(err); + assert.imageIsSimilarToFile(image, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + }.bind(this)); + } + }, + { + desc: 'should get mvt tile using buffer-size 128', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + mapConfig: createMapConfig({ mvt: 128 }), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features + assert.equal(features.length, 9); + + var map = new mapnik.Map(256, 256); + tile.render(map, new mapnik.Image(256, 256), function (err, image) { + assert.ifError(err); + assert.imageIsSimilarToFile(image, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + }.bind(this)); + } + } + ]; + + testCases.forEach(function (test) { + it(test.desc, function (done) { + var testClient = new TestClient(test.mapConfig, 1234); + var coords = test.coords; + testClient.getTile(coords.z, coords.x, coords.y, { format: test.format }, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + // tile.save(test.fixturePath); + test.assert(tile, function () { + testClient.drain(done); + }); + }); + }); + }); +}); \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png new file mode 100644 index 0000000000000000000000000000000000000000..798f7ccaf80dc1cf5c2ba2a79ee1827b85156b96 GIT binary patch literal 1379 zcmd7S|2xwO00;2TN1rCz(rtA(GM;187v?B(>hWbd?NQAQHRRLovbj57c3C1~#pBEF zC{Km3_Sq#5GK509jUHEIeK~}*4fhyp8%E_@u6@q^5%)az{^kAt?e*JR5OORCXa0pb z0021J;Xnoefa?&Y*;;+(>%%uP*qS#!;vxjvSyTLYRlhKMmoq0IzIe!18&mxUz-INEbI+{!ZdEO=kB z*fn*P9q~tWgos==qw@C9yg2qwritT>)Nfd4|2G*HJy$H(BQrP>=}w1N1P+dx)q~$S zy7i=g@<@pfN7XTnM%FD|XJq)pk#Vl7g=bn$_g!lqmBsQs!iOja-|9_`IRY5{^;)%z z>x!mLO)tHtbndg9rg-+yFDRZ#*S;MjWrs#_I0&4mx)#ot|Mq22g^gF@-gJvhA#_*U zxngs1Z7)33ler%awY;ryedbmDQeiS#`O#O-FXe?i_DPO2OEHDUKI^>fuq@NQ_lcbHStnP*y+-(+a~` zYBrS83Z*O^4O^7`wVL%9@0Yuc=DgXgI*#@& zmSFeVl=k?2bDv2fExz~B+q0e*X@Jg@dA!x#7;AFSP>KP(k`F%|bGlqM`e06( z7 zvCJ^DkIM3;=1JURV26>AzNAEje*g96T5TE?|3WwB!aSWz@GCLC^Hvm(qwUuDL02*| zc-zGZDun*h-v4Cebi5O&R_<|`ew|JOR)wayeT(-ru#3tFT_Ay6IIfY;sWMnde(W~0 z*Q3^@vGS{|4z%og@O&FJyLupi?A*dX8xOY%D`>J;(lC?WxF)E$FX-oNU!ox*( z)s|HN0O+DzkDLMkkR}BIziMd~-{{Ao000X@9dYu!`bac_4)+V$qW#UAK+6l-D25tb zcMAQ37wpUPU+4Rm4uU|_K|ti@N0WCHPQcoWQ2d^?Fwql2e>&{2FPsx&6(cSIwm^F5M;@Gr(*~*AS^M?OlEZyDw zWHvsfLY(Pc@PsAw8_+wOnU?CQkD11&PiyZh{>X4>#7p8i+t^`ot@&bF10gf*lCZH@ z_Ja%IY^vwHGu%>bWVlEKcNU?5=>z_XhS;gO2LBAFfzREA<4q)abRP7(vWTyLPHR0N zq~}&G+l$U|t!h{XUy+A?yqxqqE5S#?;8@+WBQD1UUee)hrt=8U;JkZ3xrh{MrzUYY z+XP{EkWz(s^h>hhu>K@pYJ{(a!Ee0$=wV;0`i#}VHZ?|Ka3l2c+g-69ZcuYL-b*R| zLw)s%qW2!gArkVOViPYW8CG`rDZ8S1T|22zb;nZMO^7zKks&HP;2)9Ma-!euYd(#sCNiMri2L_)p8GM(eu2758b|*#eNBn0bEo?~WNxw$R6o+SSh{ zQ%8TVUEJP0r2_crMQXMcZ;zE8YORUunu#sVjQh>1i3;2>++Es)`gM>*Ez95kkg_Ly z#+iEa8Eo#RzeXhJO7jnLjD&02O^T9j}q+25-aSgnc+N;xHVX}(LRwwz-D9W%ny_ZxTXRvTu z&8~&@3GXTk2RcV=TXJobze%T3%lRQq@{-GSb(x&d}bVklCgb)o4$Ic9U8?~UD=&vcz~FqX`y z3;H%H1l^10*<(wS&Za6CNK;fDL(SYOqPsACv56w%fXeT`wm+W%#o{oe>nokz*ws$_iW7Yl{1EOyf+7XhB=0{qu7V z@tfxI?PFE9K2Mgv=Twf>#M2SNa-?7VmDx0e;D>9!pG>UMi_10(5Q~ILe_S2IC?K^7 z^!%u^7kTNc{@638xaG<6QH^z2ZW)dPAvQ4=SZFXpz=8tmX@!3Cln5uy6|3*j81YPR zRVJH)0?)ifv|e+G=GFE;V5hwP+HT1;nu!@qi`hL8Ix@kk&IT+?HX9T!39P(i*OrBq zO^DVTb4Xvb!=Lu{9D3cmIA2NMMYy6YW{e|ez_a+H(jqvw11*tNzNvp!Yl0Ss|zr7dA9ik)!9 zCx9eH<22qh{C2UPB7j$|6S6MRwbR*@^^Onb^4NIsC~V@CRk&btA!$$h*dEZ%=aj~) z<>;-3_2Tff{nHuOGqDW|VLqYjGB#@w^2b>*1n`&cP`?FY+!fux$5u5K9BSxXK(+;m z{v(%QHN@>X{dkr_7a8uNK6P+T;aQeceVbq*Hgc`n>7uB}RI)iNjCoe)lGO?fewre; z>6Ggr4?rhUQjq=+e+d_hDQG_2$yYoa@`h_;efsuLtO*Q`oW4ejcIc8lG8<7rmp|xa z#_o@*m@PIXlgC*Dg+T9J{&PDwI`sVt=H!8v!U&SM2lloyz|PvtEp`8!iDR&F-VmPi z$|-*+<&n!Uj+M_2_OXI`H<&^ynh(-JX}hTUQ8T$=vribK-kI#hFH3DXJCl3+`D*B^H?F<8upaSLiaM=AqV2YuJ@VlhWiGO zUh5AohZXe!-wPabEIQ^KbE^5bolS)M2f+FMK5{1^DDYqhUVXC+dYLXg zX6?Q#k5lfZS`A>vvyYP#59w(gd_%()t6$Mxn5G{N8mQ;5yEB*;LcTO{TjqDD&_i}E zVl|$StWc`3qQziKjLDHK`LXeVlE}goY_9FRd*dQvn4+?w=0K-OxaIKLDD<|Xy39>+ zPjOS<-uo2d;$4UsxS49-XnjYeGy6&A`6`mUY()lDu(!WC7YAbAniSFOY+smHed)vK zDjrccWYoA2-7U;kEn-}N`i4_XSFr0m&A`PD@dXb)<4V_hyVem*^`fJIcuO)I)Vw$A zErqz?`t#_;(> Date: Mon, 24 Apr 2017 18:57:20 +0200 Subject: [PATCH 042/402] Remove filter --- test/acceptance/buffer-size-format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index bbf643bc..b311d01d 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -47,7 +47,7 @@ function createMapConfig (bufferSize) { }; } -describe.only('buffer size per format', function () { +describe('buffer size per format', function () { var testCases = [ { desc: 'should get png tile using buffer-size 1', From a6d68dba5e4f1a68477fd872e0fd2a25611dfdfe Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 19:11:53 +0200 Subject: [PATCH 043/402] Upgrades grainstore to 1.6.3 --- NEWS.md | 3 + yarn.lock | 176 +++++++++++++++++++++++++++++------------------------- 2 files changed, 98 insertions(+), 81 deletions(-) diff --git a/NEWS.md b/NEWS.md index f1464b9d..73b14e52 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.6.2 Released 2017-mm-dd +Announcements: + - Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3). + ## 3.6.1 Released 2017-04-24 diff --git a/yarn.lock b/yarn.lock index 1842800d..54bdfac7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -22,8 +22,8 @@ accepts@~1.2.12: negotiator "0.5.3" ajv@^4.9.1: - version "4.11.5" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.5.tgz#b6ee74657b993a01dce44b7944d56f485828d5bd" + version "4.11.7" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.7.tgz#8655a5d86d0824985cc471a1d913fb6729a0ec48" dependencies: co "^4.6.0" json-stable-stringify "^1.0.1" @@ -57,11 +57,11 @@ aproba@^1.0.3: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" are-we-there-yet@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz#80e470e95a084794fe1899262c5667c6e88de1b3" + version "1.1.4" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.4.tgz#bb5dca382bb94f05e15194373d16fd3ba1ca110d" dependencies: delegates "^1.0.0" - readable-stream "^2.0.0 || ^1.1.13" + readable-stream "^2.0.6" argparse@^1.0.7: version "1.0.9" @@ -151,13 +151,13 @@ boom@2.x.x: hoek "2.x.x" brace-expansion@^1.0.0: - version "1.1.6" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.6.tgz#7197d7eaa9b87e648390ea61fc66c84427420df9" + version "1.1.7" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" dependencies: balanced-match "^0.4.1" concat-map "0.0.1" -buffer-shims@^1.0.0: +buffer-shims@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" @@ -205,7 +205,7 @@ camshaft@0.54.1: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -650,8 +650,8 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" form-data@~2.1.1: - version "2.1.2" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.2.tgz#89c3534008b97eada4cbb157d58f6f5df025eae4" + version "2.1.4" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1" dependencies: asynckit "^0.4.0" combined-stream "^1.0.5" @@ -687,8 +687,8 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: rimraf "2" gauge@~2.7.1: - version "2.7.3" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.3.tgz#1c23855f962f17b3ad3d0dc7443f304542edfe09" + version "2.7.4" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" dependencies: aproba "^1.0.3" console-control-strings "^1.0.0" @@ -716,9 +716,9 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.2.tgz#886bc5bf0beb7db96e81bcbba078818de5a62683" +generic-pool@2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" generic-pool@~2.1.1: version "2.1.1" @@ -794,8 +794,8 @@ graceful-fs@~2.0.0: resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" grainstore@~1.6.0: - version "1.6.2" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.2.tgz#f5d098a8da607f23db08f3263894c8f234a8a59e" + version "1.6.3" + resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.3.tgz#6900cc811aadc1ed2c00fcd429c672f8b8e1a5cb" dependencies: carto "0.16.3" debug "~2.2.0" @@ -876,8 +876,8 @@ hoek@2.x.x: resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" hosted-git-info@^2.1.4: - version "2.4.1" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.1.tgz#4b0445e41c004a8bd1337773a4ff790ca40318c8" + version "2.4.2" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.2.tgz#0076b9f46a270506ddbaaea56496897460612a67" htmlparser2@3.8.x: version "3.8.3" @@ -1029,8 +1029,8 @@ js-base64@^2.1.9: resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" js-yaml@3.x, js-yaml@^3.4.6: - version "3.8.2" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.2.tgz#02d3e2c0f6beab20248d412c352203827d786721" + version "3.8.3" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.3.tgz#33a05ec481c850c8875929166fe1beb61c728766" dependencies: argparse "^1.0.7" esprima "^3.1.1" @@ -1210,7 +1210,7 @@ mime-db@~1.27.0: version "1.27.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1" -mime-types@^2.1.12, mime-types@~2.1.13, mime-types@~2.1.6, mime-types@~2.1.7: +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: version "2.1.15" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed" dependencies: @@ -1305,14 +1305,18 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.5.0: - version "2.5.1" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" nan@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" +nan@~2.5.0: + version "2.5.1" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" + ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" @@ -1364,8 +1368,8 @@ nopt@^4.0.1: osenv "^0.1.4" normalize-package-data@^2.3.2: - version "2.3.6" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.6.tgz#498fa420c96401f787402ba21e600def9f981fff" + version "2.3.8" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.8.tgz#d819eda2a9dedbd1ffa563ea4071d936782295bb" dependencies: hosted-git-info "^2.1.4" is-builtin-module "^1.0.0" @@ -1389,10 +1393,14 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1493,10 +1501,10 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.6.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.6.0.tgz#2e300199927b6d7db6be71e2e3435dddddf07b41" + version "1.7.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" dependencies: - generic-pool "2.4.2" + generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: @@ -1509,7 +1517,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +"pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1566,8 +1574,8 @@ postcss@5.0.19: supports-color "^3.1.2" postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8: - version "5.2.16" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.16.tgz#732b3100000f9ff8379a48a53839ed097376ad57" + version "5.2.17" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.17.tgz#cf4f597b864d65c8a492b2eabe9d706c879c388b" dependencies: chalk "^1.1.3" js-base64 "^2.1.9" @@ -1660,8 +1668,8 @@ raw-body@~2.1.5: unpipe "1.0.0" rc@^1.1.7: - version "1.1.7" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.1.7.tgz#c5ea564bb07aff9fd3a5b32e906c1d3a65940fea" + version "1.2.1" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.1.tgz#2e03e8e42ee450b8cb3dce65be1bf8974e1dfd95" dependencies: deep-extend "~0.4.0" ini "~1.3.0" @@ -1692,16 +1700,16 @@ readable-stream@1.1, readable-stream@~1.1.9: isarray "0.0.1" string_decoder "~0.10.x" -"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.1.4: - version "2.2.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.6.tgz#8b43aed76e71483938d12a8d46c6cf1a00b1f816" +readable-stream@^2.0.6, readable-stream@^2.1.4: + version "2.2.9" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" dependencies: - buffer-shims "^1.0.0" + buffer-shims "~1.0.0" core-util-is "~1.0.0" inherits "~2.0.1" isarray "~1.0.0" process-nextick-args "~1.0.6" - string_decoder "~0.10.x" + string_decoder "~1.0.0" util-deprecate "~1.0.1" readable-stream@~1.0.2: @@ -1730,32 +1738,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.69.0, request@^2.81.0: +request@2.x, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1782,6 +1765,31 @@ request@^2.69.0, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1970,8 +1978,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.11.0.tgz#2d8d5ebb4a6fab28ffba37fa62a90f4a3ea59d77" + version "1.13.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -2018,6 +2026,12 @@ string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" +string_decoder@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" + dependencies: + buffer-shims "~1.0.0" + stringstream@~0.0.4: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2084,7 +2098,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb1": version "2.3.1-cdb1" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" dependencies: @@ -2092,7 +2106,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb1: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb1": version "0.6.18-cdb1" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" dependencies: @@ -2163,15 +2177,15 @@ type-detect@^1.0.0: resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" type-is@~1.6.10, type-is@~1.6.6: - version "1.6.14" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.14.tgz#e219639c17ded1ca0789092dd54a03826b817cb2" + version "1.6.15" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410" dependencies: media-typer "0.3.0" - mime-types "~2.1.13" + mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.18" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.18.tgz#925d14bae48ab62d1883b41afe6e2261662adb8e" + version "2.8.22" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.22.tgz#d54934778a8da14903fa29a326fb24c0ab51a1a0" dependencies: source-map "~0.5.1" yargs "~3.10.0" From 93b77dc4c15e8a83c0f96643b9373a38e1d7d227 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 19:32:10 +0200 Subject: [PATCH 044/402] Release 3.6.2 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 73b14e52..ed67875b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.6.2 -Released 2017-mm-dd +Released 2017-04-24 Announcements: - Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3). From 95195fff6faa01d9a569137c54ff32550cfc216f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 24 Apr 2017 19:33:03 +0200 Subject: [PATCH 045/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ed67875b..e98dcf4f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.6.3 +Released 2017-mm-dd + + ## 3.6.2 Released 2017-04-24 diff --git a/package.json b/package.json index 8706b7f0..54388fb1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.2", + "version": "3.6.3", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 383a1a330a61a734096b28ae1fed01b4c2ab2184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Apr 2017 10:43:07 +0200 Subject: [PATCH 046/402] Test with buffer-size 0 --- test/acceptance/buffer-size-format.js | 12 ++++++------ ...er-size-1.png => tile-7.64.48-buffer-size-0.png} | Bin 2 files changed, 6 insertions(+), 6 deletions(-) rename test/fixtures/buffer-size/{tile-7.64.48-buffer-size-1.png => tile-7.64.48-buffer-size-0.png} (100%) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index b311d01d..1e84f417 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -50,11 +50,11 @@ function createMapConfig (bufferSize) { describe('buffer size per format', function () { var testCases = [ { - desc: 'should get png tile using buffer-size 1', + desc: 'should get png tile using buffer-size 0', coords: { z: 7, x: 64, y: 48 }, format: 'png', - fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png', - mapConfig: createMapConfig({ png: 1, 'grid.json': 1 }), + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + mapConfig: createMapConfig({ png: 0, 'grid.json': 0 }), assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } @@ -70,11 +70,11 @@ describe('buffer size per format', function () { } }, { - desc: 'should get mvt tile using buffer-size 1', + desc: 'should get mvt tile using buffer-size 0', coords: { z: 7, x: 64, y: 48 }, format: 'mvt', - fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png', - mapConfig: createMapConfig({ mvt: 1 }), + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + mapConfig: createMapConfig({ mvt: 0 }), assert: function (tile, callback) { var tileJSON = tile.toJSON(); var features = tileJSON[0].features; diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png similarity index 100% rename from test/fixtures/buffer-size/tile-7.64.48-buffer-size-1.png rename to test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png From 8ddccc0b0cf266ff264fd22a064c9f57a6458914 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 25 Apr 2017 12:19:49 +0200 Subject: [PATCH 047/402] Upgrades windshaft to 3.1.1 --- NEWS.md | 3 +++ package.json | 2 +- yarn.lock | 64 ++++++++++++++++++++++++++-------------------------- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/NEWS.md b/NEWS.md index e98dcf4f..6ad1d944 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.6.3 Released 2017-mm-dd +Announcements: + - Upgrades windshaft to [3.1.1](https://github.com/CartoDB/windshaft/releases/tag/3.1.1). + ## 3.6.2 Released 2017-04-24 diff --git a/package.json b/package.json index 54388fb1..f58317f7 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "3.1.0", + "windshaft": "3.1.1", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 54bdfac7..1386aaaa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -223,9 +223,9 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" @@ -1517,7 +1517,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1738,7 +1738,32 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.81.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1765,31 +1790,6 @@ request@2.x, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -2269,9 +2269,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.0.tgz#dfac2dd27a2db97e35231510743ad7db574da18f" +windshaft@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.1.tgz#5e1b20fa9bcc6f4e04f67738b18a35cb19cadbd1" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From ae9e211f309df62919f48a9dbba09589d3400f6e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 25 Apr 2017 12:35:34 +0200 Subject: [PATCH 048/402] Release 3.6.3 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6ad1d944..2fd83aed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.6.3 -Released 2017-mm-dd +Released 2017-04-25 Announcements: - Upgrades windshaft to [3.1.1](https://github.com/CartoDB/windshaft/releases/tag/3.1.1). From 567928a7f50befe44e2bd063018e518eb5edf84c Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 25 Apr 2017 12:36:29 +0200 Subject: [PATCH 049/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2fd83aed..70ee13d9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.6.4 +Released 2017-mm-dd + + ## 3.6.3 Released 2017-04-25 diff --git a/package.json b/package.json index f58317f7..07f158b7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.3", + "version": "3.6.4", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 0c08713521d615b8183bc1e78e60d3368b3fcc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Apr 2017 14:34:17 +0200 Subject: [PATCH 050/402] First attempt: support buffer-size configuration for named maps --- lib/cartodb/backends/template_maps.js | 5 + test/acceptance/buffer-size-format.js | 118 ++++++++++++++---- test/support/test-client.js | 165 +++++++++++++++++++++++++- 3 files changed, 261 insertions(+), 27 deletions(-) diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index bb728181..60a6de2c 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -474,6 +474,11 @@ TemplateMaps.prototype.instance = function(template, params) { // NOTE: we're deep-cloning the layergroup here var layergroup = JSON.parse(JSON.stringify(template.layergroup)); + + if (layergroup.buffersize) { + layergroup.buffersize = _replaceVars(layergroup.buffersize, all_params); + } + for (var i=0; i', + layers: [{ + type: 'cartodb', + options: { + sql: [ + 'select', + ' *', + 'from', + ' populated_places_simple_reduced', + ].join('\n'), + cartocss: cartocss, + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + } + } +} + +describe.only('buffer size per format for named maps', function () { + var testCases = [ + { + desc: 'should get png tile using buffer-size 0', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-buffer-size'), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + } + ]; + + testCases.forEach(function (test) { + it(test.desc, function (done) { + var testClient = new TestClient(undefined, 1234, test.template); + var coords = test.coords; + testClient.getNamedTile(coords.z, coords.x, coords.y, { format: test.format }, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + // tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-64.png'); + test.assert(tile, function () { + testClient.drain(done); + }); + }); + }); + }); +}); diff --git a/test/support/test-client.js b/test/support/test-client.js index 5998912f..578be296 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -16,9 +16,10 @@ var serverOptions = require('../../lib/cartodb/server_options'); serverOptions.analysis.batch.inlineExecution = true; var server = new CartodbWindshaft(serverOptions); -function TestClient(mapConfig, apiKey) { +function TestClient(mapConfig, apiKey, template) { this.mapConfig = mapConfig; this.apiKey = apiKey; + this.template = template; this.keysToDelete = {}; } @@ -696,3 +697,165 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba }); }); }; + +TestClient.prototype.getNamedTile = function(z, x, y, params, callback) { + if (!this.template) { + throw new Error('Template is not defined'); + } + + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + if (!params.placeholders) { + params.placeholder = {}; + } + + var urlNamed = '/api/v1/map/named'; + var url = '/api/v1/map'; + + + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + var templateName; + var layergroupId; + step( + function createTemplate () { + var next = this; + console.log(urlNamed + '?' + qs.stringify({api_key: self.apiKey})) + assert.response(server, + { + url: urlNamed + '?' + qs.stringify({api_key: self.apiKey}), + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.template) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function (res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).template_id); + } + ); + }, + function createLayergroup(err, templateId) { + var next = this; + console.log(urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey})) + assert.response(server, + { + url: urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}), + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(params.placeholders) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).layergroupid); + } + ); + }, + function getTileResult(err, _layergroupId) { + assert.ifError(err); + + var next = this; + layergroupId = _layergroupId; + + url = '/api/v1/map/' + layergroupId + '/'; + + var layers = params.layers; + + if (layers !== undefined) { + layers = Array.isArray(layers) ? layers : [layers]; + url += layers.join(',') + '/'; + } + + var format = params.format || 'png'; + + url += [z,x,y].join('/'); + url += '.' + format; + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }; + + var expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + var isPng = format.match(/png$/); + + if (isPng) { + request.encoding = 'binary'; + expectedResponse.headers['Content-Type'] = 'image/png'; + } + + var isMvt = format.match(/mvt$/); + + if (isMvt) { + request.encoding = 'binary'; + expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; + } + + assert.response(server, request, expectedResponse, function(res, err) { + assert.ifError(err); + + var obj; + + if (isPng) { + obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + } + else if (isMvt) { + obj = new mapnik.VectorTile(z, x, y); + obj.setDataSync(new Buffer(res.body, 'binary')); + } + + else { + obj = JSON.parse(res.body); + } + + next(null, res, obj); + }); + }, + function finish(err, res, image) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + return callback(err, res, image); + } + ); +}; From f29ee1b4aca88f066fd53e2f7d14da363168203a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Apr 2017 15:48:23 +0200 Subject: [PATCH 051/402] Add test to use placeholder buffer-size value --- test/acceptance/buffer-size-format.js | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 82b83fff..6df3914c 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -118,7 +118,7 @@ describe('buffer size per format', function () { testClient.getTile(coords.z, coords.x, coords.y, { format: test.format }, function (err, res, tile) { assert.ifError(err); // To generate images use: - tile.save(test.fixturePath); + // tile.save(test.fixturePath); test.assert(tile, function () { testClient.drain(done); }); @@ -159,14 +159,25 @@ function createBufferSizeTemplate (name, cartocss) { } } -describe.only('buffer size per format for named maps', function () { +describe('buffer size per format for named maps', function () { var testCases = [ { - desc: 'should get png tile using buffer-size 0', + desc: 'should get png tile using buffer-size 0 (default value in template)', coords: { z: 7, x: 64, y: 48 }, format: 'png', fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', - template: createBufferSizeTemplate('named-buffer-size'), + template: createBufferSizeTemplate('named-default-buffer-size'), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128 (placehoder value)', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { buffersize: 128 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + template: createBufferSizeTemplate('named-custom-buffer-size'), assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } @@ -177,7 +188,11 @@ describe.only('buffer size per format for named maps', function () { it(test.desc, function (done) { var testClient = new TestClient(undefined, 1234, test.template); var coords = test.coords; - testClient.getNamedTile(coords.z, coords.x, coords.y, { format: test.format }, function (err, res, tile) { + var options = { + format: test.format, + placeholders: test.placeholders + } + testClient.getNamedTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: // tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-64.png'); From 0577fa530821c3d85bda3b61fcba90d86ad40135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Apr 2017 17:54:31 +0200 Subject: [PATCH 052/402] Add test --- test/acceptance/buffer-size-format.js | 65 +++++++++++++++++---------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 6df3914c..80a75a24 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -26,7 +26,7 @@ var CARTOCSS_LABELS = [ ' text-placement: point;', ' text-placement-type: dummy;', '}' -].join('\n') +].join('\n'); function createMapConfig (bufferSize, cartocss) { cartocss = cartocss || CARTOCSS_LABELS; @@ -99,7 +99,7 @@ describe('buffer size per format', function () { mapConfig: createMapConfig({ mvt: 128 }), assert: function (tile, callback) { var tileJSON = tile.toJSON(); - var features = tileJSON[0].features + var features = tileJSON[0].features; assert.equal(features.length, 9); var map = new mapnik.Map(256, 256); @@ -127,39 +127,23 @@ describe('buffer size per format', function () { }); }); -function createBufferSizeTemplate (name, cartocss) { +function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) { cartocss = cartocss || CARTOCSS_LABELS; + return { "version": "0.0.1", "name": name, - "placeholders": { + "placeholders": placeholders || { "buffersize": { "type": "number", "default": "0" } }, - "layergroup": { - version: '1.6.0', - buffersize: '<%= buffersize %>', - layers: [{ - type: 'cartodb', - options: { - sql: [ - 'select', - ' *', - 'from', - ' populated_places_simple_reduced', - ].join('\n'), - cartocss: cartocss, - cartocss_version: '2.3.0', - interactivity: 'cartodb_id' - } - }] - } + "layergroup": createMapConfig(buffersize) } } -describe('buffer size per format for named maps', function () { +describe.only('buffer size per format for named maps', function () { var testCases = [ { desc: 'should get png tile using buffer-size 0 (default value in template)', @@ -181,6 +165,41 @@ describe('buffer size per format for named maps', function () { assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } + }, + { + desc: 'should get png tile using buffer-size 0 (default value in template by format)', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-default-buffer-size-by-format', { + png: '<%= buffersize_png %>' + }, { + "buffersize_png": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { buffersize: 128 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + template: createBufferSizeTemplate('named-custom-buffer-size-by-format', { + png: '<%= buffersize_png %>' + }, { + "buffersize_png": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } } ]; From 7ea7a991aa2fc26470e72ead739d27a755a10cc2 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 25 Apr 2017 19:27:31 +0200 Subject: [PATCH 053/402] Buffersize customizable through named maps' placeholders --- lib/cartodb/backends/template_maps.js | 21 ++++++++++++---- test/acceptance/buffer-size-format.js | 36 ++++++++++++++------------- test/support/test-client.js | 10 ++++---- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index 60a6de2c..5d856976 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -296,7 +296,7 @@ TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) { // @param callback function(err) // TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) { - + var self = this; template = templateDefaults(template); @@ -430,13 +430,15 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/, _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/; function _replaceVars (str, params) { - //return _.template(str, params); // lazy way, possibly dangerous // Construct regular expressions for each param - Object.keys(params).forEach(function(k) { + Object.keys(params).forEach(function(k) { str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]); }); return str; } +function isDictionary(val) { + return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val)); +} TemplateMaps.prototype.instance = function(template, params) { var all_params = {}; var phold = template.placeholders || {}; @@ -465,6 +467,11 @@ TemplateMaps.prototype.instance = function(template, params) { throw new Error("Invalid css_color value for template parameter '" + k + "': " + val); } } + else if ( type === 'dictionary' ) { + if (!isDictionary(val)) { + throw new Error("Invalid dictionary value for template parameter '" + k + "': " + val); + } + } else { // NOTE: should be checked at template create/update time throw new Error("Invalid placeholder type '" + type + "'"); @@ -475,8 +482,12 @@ TemplateMaps.prototype.instance = function(template, params) { // NOTE: we're deep-cloning the layergroup here var layergroup = JSON.parse(JSON.stringify(template.layergroup)); - if (layergroup.buffersize) { - layergroup.buffersize = _replaceVars(layergroup.buffersize, all_params); + if (typeof layergroup.buffersize === 'string') { + layergroup.buffersize = Number(_replaceVars(layergroup.buffersize, all_params)); + } else if (isDictionary(layergroup.buffersize)) { + Object.keys(layergroup.buffersize).forEach(function(k) { + layergroup.buffersize[k] = Number(_replaceVars(layergroup.buffersize[k], all_params)); + }); } for (var i=0; i'), assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } @@ -161,7 +161,7 @@ describe.only('buffer size per format for named maps', function () { format: 'png', placeholders: { buffersize: 128 }, fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', - template: createBufferSizeTemplate('named-custom-buffer-size'), + template: createBufferSizeTemplate('named-custom-buffer-size', '<%= buffersize %>'), assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } @@ -170,9 +170,10 @@ describe.only('buffer size per format for named maps', function () { desc: 'should get png tile using buffer-size 0 (default value in template by format)', coords: { z: 7, x: 64, y: 48 }, format: 'png', + placeholders: { buffersize_png: 0 }, fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', - template: createBufferSizeTemplate('named-default-buffer-size-by-format', { - png: '<%= buffersize_png %>' + template: createBufferSizeTemplate('named-default-buffer-size-by-format', { + png: '<%= buffersize_png %>' }, { "buffersize_png": { "type": "number", @@ -187,10 +188,10 @@ describe.only('buffer size per format for named maps', function () { desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)', coords: { z: 7, x: 64, y: 48 }, format: 'png', - placeholders: { buffersize: 128 }, + placeholders: { buffersize_png: 128 }, fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', - template: createBufferSizeTemplate('named-custom-buffer-size-by-format', { - png: '<%= buffersize_png %>' + template: createBufferSizeTemplate('named-custom-buffer-size-by-format', { + png: '<%= buffersize_png %>' }, { "buffersize_png": { "type": "number", @@ -207,15 +208,16 @@ describe.only('buffer size per format for named maps', function () { it(test.desc, function (done) { var testClient = new TestClient(undefined, 1234, test.template); var coords = test.coords; - var options = { - format: test.format, - placeholders: test.placeholders - } + var options = { + format: test.format, + placeholders: test.placeholders + }; testClient.getNamedTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: - // tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-64.png'); - test.assert(tile, function () { + //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png'); + test.assert(tile, function (err) { + assert.ifError(err); testClient.drain(done); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 578be296..3e3edf55 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -504,8 +504,8 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { else if (isMvt) { obj = new mapnik.VectorTile(z, x, y); obj.setDataSync(new Buffer(res.body, 'binary')); - } - + } + else { obj = JSON.parse(res.body); } @@ -711,7 +711,7 @@ TestClient.prototype.getNamedTile = function(z, x, y, params, callback) { } if (!params.placeholders) { - params.placeholder = {}; + params.placeholders = {}; } var urlNamed = '/api/v1/map/named'; @@ -843,8 +843,8 @@ TestClient.prototype.getNamedTile = function(z, x, y, params, callback) { else if (isMvt) { obj = new mapnik.VectorTile(z, x, y); obj.setDataSync(new Buffer(res.body, 'binary')); - } - + } + else { obj = JSON.parse(res.body); } From 07e507e1aa34311539e2b4e67b185ab683ff8089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Apr 2017 19:40:12 +0200 Subject: [PATCH 054/402] Remove dictionary as placeholder type for named maps --- lib/cartodb/backends/template_maps.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index 5d856976..812375e6 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -436,9 +436,11 @@ function _replaceVars (str, params) { }); return str; } -function isDictionary(val) { + +function isObject(val) { return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val)); } + TemplateMaps.prototype.instance = function(template, params) { var all_params = {}; var phold = template.placeholders || {}; @@ -467,11 +469,6 @@ TemplateMaps.prototype.instance = function(template, params) { throw new Error("Invalid css_color value for template parameter '" + k + "': " + val); } } - else if ( type === 'dictionary' ) { - if (!isDictionary(val)) { - throw new Error("Invalid dictionary value for template parameter '" + k + "': " + val); - } - } else { // NOTE: should be checked at template create/update time throw new Error("Invalid placeholder type '" + type + "'"); @@ -484,7 +481,7 @@ TemplateMaps.prototype.instance = function(template, params) { if (typeof layergroup.buffersize === 'string') { layergroup.buffersize = Number(_replaceVars(layergroup.buffersize, all_params)); - } else if (isDictionary(layergroup.buffersize)) { + } else if (isObject(layergroup.buffersize)) { Object.keys(layergroup.buffersize).forEach(function(k) { layergroup.buffersize[k] = Number(_replaceVars(layergroup.buffersize[k], all_params)); }); From 0d840e6dafe6aeef23af579bdcfebabde84590cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Apr 2017 19:41:30 +0200 Subject: [PATCH 055/402] Javascript style typo --- test/acceptance/buffer-size-format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 67b1ae09..0780af84 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -140,7 +140,7 @@ function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) { } }, "layergroup": createMapConfig(buffersize) - } + }; } describe('buffer size per format for named maps', function () { From 40c0e306af86fe91f3afb6d31a94dfc6517db16e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Apr 2017 20:40:17 +0200 Subject: [PATCH 056/402] Remove invalid assertions --- test/acceptance/buffer-size-format.js | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 0780af84..b8f40766 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -3,7 +3,6 @@ require('../support/test_helper'); var assert = require('../support/assert'); var TestClient = require('../support/test-client'); var IMAGE_TOLERANCE_PER_MIL = 5; -var mapnik = require('windshaft').mapnik; var CARTOCSS_LABELS = [ '#layer {', @@ -83,12 +82,7 @@ describe('buffer size per format', function () { var tileJSON = tile.toJSON(); var features = tileJSON[0].features; assert.equal(features.length, 1); - - var map = new mapnik.Map(256, 256); - tile.render(map, new mapnik.Image(256, 256), function (err, image) { - assert.ifError(err); - assert.imageIsSimilarToFile(image, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); - }.bind(this)); + callback(); } }, { @@ -101,12 +95,7 @@ describe('buffer size per format', function () { var tileJSON = tile.toJSON(); var features = tileJSON[0].features; assert.equal(features.length, 9); - - var map = new mapnik.Map(256, 256); - tile.render(map, new mapnik.Image(256, 256), function (err, image) { - assert.ifError(err); - assert.imageIsSimilarToFile(image, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); - }.bind(this)); + callback(); } } ]; @@ -119,7 +108,8 @@ describe('buffer size per format', function () { assert.ifError(err); // To generate images use: // tile.save(test.fixturePath); - test.assert(tile, function () { + test.assert(tile, function (err) { + assert.ifError(err); testClient.drain(done); }); }); From c481d6473c1cde702b3ce2d9642fede922918557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 26 Apr 2017 17:01:21 +0200 Subject: [PATCH 057/402] Use parseInt instead of number constructor --- lib/cartodb/backends/template_maps.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index 812375e6..66d64dda 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -430,8 +430,8 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/, _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/; function _replaceVars (str, params) { - // Construct regular expressions for each param - Object.keys(params).forEach(function(k) { + // Construct regular expressions for each param + Object.keys(params).forEach(function(k) { str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]); }); return str; @@ -480,10 +480,10 @@ TemplateMaps.prototype.instance = function(template, params) { var layergroup = JSON.parse(JSON.stringify(template.layergroup)); if (typeof layergroup.buffersize === 'string') { - layergroup.buffersize = Number(_replaceVars(layergroup.buffersize, all_params)); + layergroup.buffersize = parseInt(_replaceVars(layergroup.buffersize, all_params), 10); } else if (isObject(layergroup.buffersize)) { Object.keys(layergroup.buffersize).forEach(function(k) { - layergroup.buffersize[k] = Number(_replaceVars(layergroup.buffersize[k], all_params)); + layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10); }); } From 2f4e4246a4ba05bda8026086d263d01a16105a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 26 Apr 2017 18:27:18 +0200 Subject: [PATCH 058/402] Refactor test-client in order to use same interface for named and anonymous maps --- test/acceptance/buffer-size-format.js | 4 +- test/support/test-client.js | 226 +++++++------------------- 2 files changed, 60 insertions(+), 170 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index b8f40766..50a11329 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -196,13 +196,13 @@ describe('buffer size per format for named maps', function () { testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(undefined, 1234, test.template); + var testClient = new TestClient(test.template, 1234); var coords = test.coords; var options = { format: test.format, placeholders: test.placeholders }; - testClient.getNamedTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png'); diff --git a/test/support/test-client.js b/test/support/test-client.js index 3e3edf55..b2a326b8 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -16,15 +16,23 @@ var serverOptions = require('../../lib/cartodb/server_options'); serverOptions.analysis.batch.inlineExecution = true; var server = new CartodbWindshaft(serverOptions); -function TestClient(mapConfig, apiKey, template) { - this.mapConfig = mapConfig; +function TestClient(config, apiKey) { + this.mapConfig = isMapConfig(config) ? config : null; + this.template = isTemplate(config) ? config : null; this.apiKey = apiKey; - this.template = template; this.keysToDelete = {}; } module.exports = TestClient; +function isMapConfig(config) { + return config && config.layers; +} + +function isTemplate(config) { + return config && config.layergroup; +} + module.exports.RESPONSE = { ERROR: { status: 400, @@ -407,6 +415,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } var url = '/api/v1/map'; + var urlNamed = url + '/named'; if (this.apiKey) { url += '?' + qs.stringify({api_key: this.apiKey}); @@ -414,17 +423,60 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { var layergroupId; step( - function createLayergroup() { + function createTemplate () { var next = this; + + if (!self.template) { + return next(); + } + + if (!self.apiKey) { + return next(new Error('apiKey param is mandatory to create a new template')); + } + + params.placeholders = params.placeholders || {}; + assert.response(server, { - url: url, + url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), method: 'POST', headers: { host: 'localhost', 'Content-Type': 'application/json' }, - data: JSON.stringify(self.mapConfig) + data: JSON.stringify(self.template) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function (res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).template_id); + } + ); + }, + function createLayergroup(err, templateId) { + var next = this; + + var data = templateId ? params.placeholders : self.mapConfig + var path = templateId ? + urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : + url; + + assert.response(server, + { + url: path, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(data) }, { status: 200, @@ -697,165 +749,3 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba }); }); }; - -TestClient.prototype.getNamedTile = function(z, x, y, params, callback) { - if (!this.template) { - throw new Error('Template is not defined'); - } - - var self = this; - - if (!callback) { - callback = params; - params = {}; - } - - if (!params.placeholders) { - params.placeholders = {}; - } - - var urlNamed = '/api/v1/map/named'; - var url = '/api/v1/map'; - - - if (this.apiKey) { - url += '?' + qs.stringify({api_key: this.apiKey}); - } - - var templateName; - var layergroupId; - step( - function createTemplate () { - var next = this; - console.log(urlNamed + '?' + qs.stringify({api_key: self.apiKey})) - assert.response(server, - { - url: urlNamed + '?' + qs.stringify({api_key: self.apiKey}), - method: 'POST', - headers: { - host: 'localhost', - 'Content-Type': 'application/json' - }, - data: JSON.stringify(self.template) - }, - { - status: 200, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }, - function (res, err) { - if (err) { - return next(err); - } - return next(null, JSON.parse(res.body).template_id); - } - ); - }, - function createLayergroup(err, templateId) { - var next = this; - console.log(urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey})) - assert.response(server, - { - url: urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}), - method: 'POST', - headers: { - host: 'localhost', - 'Content-Type': 'application/json' - }, - data: JSON.stringify(params.placeholders) - }, - { - status: 200, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }, - function(res, err) { - if (err) { - return next(err); - } - return next(null, JSON.parse(res.body).layergroupid); - } - ); - }, - function getTileResult(err, _layergroupId) { - assert.ifError(err); - - var next = this; - layergroupId = _layergroupId; - - url = '/api/v1/map/' + layergroupId + '/'; - - var layers = params.layers; - - if (layers !== undefined) { - layers = Array.isArray(layers) ? layers : [layers]; - url += layers.join(',') + '/'; - } - - var format = params.format || 'png'; - - url += [z,x,y].join('/'); - url += '.' + format; - - if (self.apiKey) { - url += '?' + qs.stringify({api_key: self.apiKey}); - } - - var request = { - url: url, - method: 'GET', - headers: { - host: 'localhost' - } - }; - - var expectedResponse = { - status: 200, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - var isPng = format.match(/png$/); - - if (isPng) { - request.encoding = 'binary'; - expectedResponse.headers['Content-Type'] = 'image/png'; - } - - var isMvt = format.match(/mvt$/); - - if (isMvt) { - request.encoding = 'binary'; - expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; - } - - assert.response(server, request, expectedResponse, function(res, err) { - assert.ifError(err); - - var obj; - - if (isPng) { - obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } - else if (isMvt) { - obj = new mapnik.VectorTile(z, x, y); - obj.setDataSync(new Buffer(res.body, 'binary')); - } - - else { - obj = JSON.parse(res.body); - } - - next(null, res, obj); - }); - }, - function finish(err, res, image) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - return callback(err, res, image); - } - ); -}; From 332a56b7360324fee3adcb12cf89304db2291a92 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 28 Apr 2017 14:22:16 +0200 Subject: [PATCH 059/402] Mapconfig only support object for the buffer-size property --- .gitignore | 3 +-- lib/cartodb/backends/template_maps.js | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b39f2801..83bce537 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,5 @@ tools/munin/windshaft.conf logs/ pids/ redis.pid -test.log -npm-debug.log +*.log coverage/ diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index 66d64dda..75a482fe 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -479,9 +479,7 @@ TemplateMaps.prototype.instance = function(template, params) { // NOTE: we're deep-cloning the layergroup here var layergroup = JSON.parse(JSON.stringify(template.layergroup)); - if (typeof layergroup.buffersize === 'string') { - layergroup.buffersize = parseInt(_replaceVars(layergroup.buffersize, all_params), 10); - } else if (isObject(layergroup.buffersize)) { + if (layergroup.buffersize && isObject(layergroup.buffersize)) { Object.keys(layergroup.buffersize).forEach(function(k) { layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10); }); From bfb283c5ba2c5027e2a952b6997df73010212ce3 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 28 Apr 2017 14:46:36 +0200 Subject: [PATCH 060/402] wip --- .../adapter/mapconfig-named-layers-adapter.js | 1 - .../mapconfig/adapter/mapconfig-named-map-adapter.js | 11 +++++++++++ lib/cartodb/server.js | 1 + test/acceptance/buffer-size-format.js | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js index fd00e344..17c2059f 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js @@ -43,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC if (nestedNamedLayers.length > 0) { var nestedNamedMapsError = new Error('Nested named layers are not allowed'); - // nestedNamedMapsError.http_status = 400; return done(nestedNamedMapsError); } diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js new file mode 100644 index 00000000..605ea5f5 --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js @@ -0,0 +1,11 @@ +var queue = require('queue-async'); +var _ = require('underscore'); +var Datasource = require('windshaft').model.Datasource; + +function MapConfigNamedMapAdapter() { +} + +module.exports = MapConfigNamedMapAdapter; + +MapConfigNamedMapAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index b695a367..b04229a1 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -155,6 +155,7 @@ module.exports = function(serverOptions) { var mapConfigAdapter = new MapConfigAdapter( new MapConfigNamedLayersAdapter(templateMaps, pgConnection), + new MapConfigNamedMapAdapter(), new SqlWrapMapConfigAdapter(), new DataviewsWidgetsAdapter(), new AnalysisMapConfigAdapter(analysisBackend), diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 50a11329..d6c44964 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -140,7 +140,7 @@ describe('buffer size per format for named maps', function () { coords: { z: 7, x: 64, y: 48 }, format: 'png', fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', - template: createBufferSizeTemplate('named-default-buffer-size', '<%= buffersize %>'), + template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}), assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } From be58adb1b9f64126e4407a6610a751eab139f55d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 28 Apr 2017 14:46:36 +0200 Subject: [PATCH 061/402] Be able to override buffer-size configuration without placeholders in named maps --- .../adapter/mapconfig-named-layers-adapter.js | 1 - .../adapter/mapconfig-named-map-adapter.js | 35 ++++ .../mapconfig/provider/named-map-provider.js | 1 + lib/cartodb/server.js | 2 + test/acceptance/buffer-size-format.js | 182 +++++++++++++++++- .../tile-7.64.48-buffer-size-128.geojson | 1 + .../tile-7.64.48-buffer-size-128.grid.json | 1 + ...-grid.json.7.64.48-buffer-size-0.grid.json | 1 + .../tile-mvt-7.64.48-buffer-size-0.geojson | 1 + .../tile-mvt-7.64.48-buffer-size-0.mvt | Bin 0 -> 615 bytes .../tile-mvt-7.64.48-buffer-size-128.mvt | Bin 0 -> 1877 bytes test/support/test-client.js | 15 +- 12 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js create mode 100644 test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson create mode 100644 test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json create mode 100644 test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json create mode 100644 test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson create mode 100644 test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt create mode 100644 test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js index fd00e344..17c2059f 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js @@ -43,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC if (nestedNamedLayers.length > 0) { var nestedNamedMapsError = new Error('Nested named layers are not allowed'); - // nestedNamedMapsError.http_status = 400; return done(nestedNamedMapsError); } diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js new file mode 100644 index 00000000..76e8f3f5 --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js @@ -0,0 +1,35 @@ +var _ = require('underscore'); + +function MapConfigNamedMapAdapter() { +} + +module.exports = MapConfigNamedMapAdapter; + +MapConfigNamedMapAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { + if (context.templateParams && + context.templateParams.buffersize && + isValidBufferSize(context.templateParams.buffersize)) { + requestMapConfig.buffersize = context.templateParams.buffersize; + } + + process.nextTick(function () { + callback(null, requestMapConfig); + }); +}; + +function isValidBufferSize (bufferSize) { + var formats = ['png', 'png32', 'mvt', 'grid.json', 'geojson']; + + if (!_.isObject(bufferSize) || (_.isArray(bufferSize) || _.isFunction(bufferSize))) { + return false; + } + + for (var index = 0; index < formats.length; index++) { + var bufferSizeByFormat = bufferSize[formats[index]]; + if (bufferSizeByFormat && !Number.isFinite(bufferSizeByFormat)) { + return false; + } + } + + return true; +} diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 2439a650..594ec247 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -90,6 +90,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { }, function instantiateTemplate(err, templateParams) { assert.ifError(err); + context.templateParams = templateParams; return self.templateMaps.instance(self.template, templateParams); }, function prepareAdapterMapConfig(err, requestMapConfig) { diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index b695a367..22b14f5b 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -35,6 +35,7 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter'); +var MapConfigNamedMapAdapter = require('./models/mapconfig/adapter/mapconfig-named-map-adapter'); var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter'); var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); @@ -155,6 +156,7 @@ module.exports = function(serverOptions) { var mapConfigAdapter = new MapConfigAdapter( new MapConfigNamedLayersAdapter(templateMaps, pgConnection), + new MapConfigNamedMapAdapter(), new SqlWrapMapConfigAdapter(), new DataviewsWidgetsAdapter(), new AnalysisMapConfigAdapter(analysisBackend), diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 50a11329..8c8f6f7b 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -1,7 +1,9 @@ require('../support/test_helper'); +var fs = require('fs'); var assert = require('../support/assert'); var TestClient = require('../support/test-client'); +var mapnik = require('windshaft').mapnik; var IMAGE_TOLERANCE_PER_MIL = 5; var CARTOCSS_LABELS = [ @@ -76,7 +78,7 @@ describe('buffer size per format', function () { desc: 'should get mvt tile using buffer-size 0', coords: { z: 7, x: 64, y: 48 }, format: 'mvt', - fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.mvt', mapConfig: createMapConfig({ mvt: 0 }), assert: function (tile, callback) { var tileJSON = tile.toJSON(); @@ -89,7 +91,7 @@ describe('buffer size per format', function () { desc: 'should get mvt tile using buffer-size 128', coords: { z: 7, x: 64, y: 48 }, format: 'mvt', - fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.mvt', mapConfig: createMapConfig({ mvt: 128 }), assert: function (tile, callback) { var tileJSON = tile.toJSON(); @@ -140,7 +142,7 @@ describe('buffer size per format for named maps', function () { coords: { z: 7, x: 64, y: 48 }, format: 'png', fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', - template: createBufferSizeTemplate('named-default-buffer-size', '<%= buffersize %>'), + template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}), assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } @@ -151,7 +153,7 @@ describe('buffer size per format for named maps', function () { format: 'png', placeholders: { buffersize: 128 }, fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', - template: createBufferSizeTemplate('named-custom-buffer-size', '<%= buffersize %>'), + template: createBufferSizeTemplate('named-custom-buffer-size', { png: '<%= buffersize %>'}), assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } @@ -214,3 +216,175 @@ describe('buffer size per format for named maps', function () { }); }); }); + + +describe('buffer size per format for named maps w/o placeholders', function () { + var testCases = [ + { + desc: 'should get png tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { + buffersize: { + png: 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-no-buffer-size-png-0', {}, {}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { + buffersize: { + png: 128 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + template: createBufferSizeTemplate('named-no-buffer-size-png-128', {}, {}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get mvt tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + placeholders: { + buffersize: { + mvt: 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt', + template: createBufferSizeTemplate('named-no-buffer-size-mvt', {}, {}), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features; + + var dataFixture = fs.readFileSync(this.fixturePath); + var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y); + vtile.setDataSync(dataFixture); + var vtileJSON = vtile.toJSON(); + var vtileFeatures = vtileJSON[0].features; + + assert.equal(features.length, vtileFeatures.length); + callback(); + } + }, + { + desc: 'should get mvt tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + placeholders: { + buffersize: { + mvt: 128 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt', + template: createBufferSizeTemplate('named-no-buffer-size-mvt-128', {}, {}), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features; + + var dataFixture = fs.readFileSync(this.fixturePath); + var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y); + vtile.setDataSync(dataFixture); + var vtileJSON = vtile.toJSON(); + var vtileFeatures = vtileJSON[0].features; + + assert.equal(features.length, vtileFeatures.length); + callback(); + } + }, + { + desc: 'should get geojson tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'geojson', + placeholders: { + buffersize: { + geojson: 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson', + template: createBufferSizeTemplate('named-no-buffer-size-geojson-0', {}, {}), + assert: function (tile, callback) { + var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); + assert.equal(tile.features.length, dataFixture.features.length); + callback(); + } + }, + { + desc: 'should get geojson tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'geojson', + placeholders: { + buffersize: { + geojson: 128 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson', + template: createBufferSizeTemplate('named-no-buffer-size-geojson-128', {}, {}), + assert: function (tile, callback) { + var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); + assert.equal(tile.features.length, dataFixture.features.length); + callback(); + } + }, + { + desc: 'should get grid.json tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + placeholders: { + buffersize: { + 'grid.json': 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', + template: createBufferSizeTemplate('named-no-buffer-size-grid-json-0', {}, {}), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback); + } + }, + { + desc: 'should get grid.json tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + placeholders: { + buffersize: { + 'grid.json': 128 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', + template: createBufferSizeTemplate('named-no-buffer-size-grid-json-128', {}, {}), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); + } + }, + ]; + + testCases.forEach(function (test) { + it(test.desc, function (done) { + var testClient = new TestClient(test.template, 1234); + var coords = test.coords; + var options = { + format: test.format, + placeholders: test.placeholders + }; + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + //tile.save(test.fixturePath); + // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); + // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); + test.assert(tile, function (err) { + assert.ifError(err); + testClient.drain(done); + }); + }); + }); + }); +}); diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson new file mode 100644 index 00000000..e89c71dd --- /dev/null +++ b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[-53839,4629161]},"properties":{"name":"Alicante","cartodb_id":1200}},{"type":"Feature","geometry":{"type":"Point","coordinates":[242835,5069332]},"properties":{"name":"Barcelona","cartodb_id":5330}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-5567,4861644]},"properties":{"name":"Castello","cartodb_id":1201}},{"type":"Feature","geometry":{"type":"Point","coordinates":[272735,5092314]},"properties":{"name":"Mataro","cartodb_id":615}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125787,4576600]},"properties":{"name":"Murcia","cartodb_id":952}},{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139148,5030112]},"properties":{"name":"Tarragona","cartodb_id":616}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-44746,4791667]},"properties":{"name":"Valencia","cartodb_id":5942}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-99072,5108695]},"properties":{"name":"Zaragoza","cartodb_id":5932}}]} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json new file mode 100644 index 00000000..c06b8d91 --- /dev/null +++ b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json @@ -0,0 +1 @@ +{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !! ","!!! !!!!! ","!!!!!!! ! ","!!! !!!!! "," !! ! "," "," "," "," "," "," "," "," ### # "," ####### ###"," ####### ## ","$ ## #### ## ","$$ ","$$ ","$$ "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","9","2","1"],"data":{"1":{"cartodb_id":5942},"2":{"cartodb_id":5500},"9":{"cartodb_id":1201}}} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json b/test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json new file mode 100644 index 00000000..24581bc3 --- /dev/null +++ b/test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json @@ -0,0 +1 @@ +{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !!! ! "," !!!!!!! !!!"," !!!!!!! !! "," !! !!!! !! "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","1"],"data":{"1":{"cartodb_id":5500}}} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson new file mode 100644 index 00000000..500fb3ea --- /dev/null +++ b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}}]} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt new file mode 100644 index 0000000000000000000000000000000000000000..b87f1b9a6d5d6a4922b1d822d2f71e5e0fcebe85 GIT binary patch literal 615 zcmXw0%We}f6wPGbZkxxX3C#s6%iRDHsD&j63WWt!h)Vr{=yt|Qta|Kd#)W3r?veNb z{(%iYz>)=@zz%)^HY~6w?JPz<=iGabk1zZR&f6rCxa8IDoeK&v7Uk{)AF+`5ua1va zIL8GGhb{X-;&g>2XE6zn7bsLi?KM=7bY9mIYx)8u$BNew*;bSWNRE}!#mPhz%%}hE zL7dM~&3Mtwy?Ev@&rj{xHf=##i0KsTTnI?qQE9{3kPHr1J+#j4d|IJ8eE?+2(VSI0 zm&p30`@ATHy1z%QuP`VfopJ5%W_`n%Rtm8Qg=+D2SD=3tO-#+yq zb*eUVURmPuqa~Wot7H)*wSK=^Tt<3_Ld^jTbX-tx}be(Lhb=JEZy-vTIMT72UZ`d70H~L%IcmSI> Lhqp$zTif1$xTB}x literal 0 HcmV?d00001 diff --git a/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt new file mode 100644 index 0000000000000000000000000000000000000000..c6e5421217b66b22d078be2ea92910bfda79b91e GIT binary patch literal 1877 zcmYk6Yitx%7>2u-*}1e#x4YAJ=M>5~pil&4TLpzew_PrBlR^;zFYRGwx}7jPvzeV0 zN{j*dp%DoZLuye%3=%Y&0Nx;x*eYtGsDTnCf)e~;M1z`uR2m`powM5s`{Ubl&YV5- zyzl#+MML7|pb)Weui?&^yj4Lg*0VD@ag)jjp#>f7#kq!aDpOqsAH-DIZNotk#3=jG>c( zp>2dygGw3Vb+fx0=QYzrLbrjvf@^3NR;z@{Ry}Ll(~=6DXL{H|e8zCdo8dgBnroDo zpop1wI&q^K7J@*}TX=(Ex!B$a2Zs=6VbAmmx}oy&vu93^ss1FLf<2C%8C^YeXxI2} z`hOgNt@+L@_Ou@OVF0l(qYBFfepFW@;<82tYo^y*9MKJez>RJ|VX}gFQj2OhG9=;< zjJ&)C4#Dv@VslOo-nLl{D}!$(OHG4;Rh$l}Sug7pS!&gf27ak`aaG7)$RSt5Ru<61f)jV(?DREFQp4^HI*vF5U}Kp+?lg~9E$1ruF@jiLSrHvmRS^^8q9j%`HI=o|M3o#vd|iBO*|>lj6~x7P3+I(K(1Bee z51HPaLK|gk3*v}Q+wlywT8O8MF)I{YA>`9xpNHiokc8d)ZEo9F=@blKpKy4^uXC6b zX|8r`Wieu@dzPohg(|wD%r}(f*s>f~!(d)Lc=GeVHBdj@6@|L(r6{aMVoRyc51NM? zxL|BvNv8xlt0Of;XMHUwO;Dw>h(Kvw6%(%;KUS&(rQ>Sr6XRtC-BnTwsi!gYTjAtU z3?hdbiXv0+b!m7W(Qv;0%Js^l^N9wr>5U67ZVIH-3PEVYu4X`g;iL*@_%k@;-Jr@S zg9qU&ti1Ml7u3EFN2!OB`zyGRPpty=gvdmn`ff^nk2ndaCqq6B@dl{@sHfE4o0uw3 zLlu;|hB&s$$e1VJw-p^cE4XkO;myZcz*c&w_4|GwF>|-t2=S`|D*1>f!IJO5JloKCzYy``t4g`0l&I)ftp8DK-LM6S&$WHBnFRPdp&c^gRtJ zzlF(_Zg+A8xR!>ljSTH4n&ylmVT)Y1btW*6S<52Yv@1mQZ6u>efB`pTg` zzmtX9zoQ0Lx!59_{H0kVQI3htE+tPqoK@=Z2Ze_y;cRgZ5H1w*6wZzi;o`)RZ*Ndj|Y{z{{W8PP*s2LDES- oe8!hRb_e#>bG#2if_l;id5VH`id_JL0R&5!f^0}=vOWs(KV!GHW&i*H literal 0 HcmV?d00001 diff --git a/test/support/test-client.js b/test/support/test-client.js index b2a326b8..bbb32c9d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -545,6 +545,20 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; } + var isGeojson = format.match(/geojson$/); + + if (isGeojson) { + request.encoding = 'utf-8'; + expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + + var isGridJSON = format.match(/grid.json$/); + + if (isGridJSON) { + request.encoding = 'utf-8'; + expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + assert.response(server, request, expectedResponse, function(res, err) { assert.ifError(err); @@ -557,7 +571,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { obj = new mapnik.VectorTile(z, x, y); obj.setDataSync(new Buffer(res.body, 'binary')); } - else { obj = JSON.parse(res.body); } From c9af38ecd0e259e8c35102ea16e1e8436736aef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Apr 2017 19:21:51 +0200 Subject: [PATCH 062/402] Fix issue when 'grid.json' format is not captured properly due to a weird behaviour in regex --- lib/cartodb/controllers/layergroup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 79268d6e..fb1aab9b 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -56,7 +56,7 @@ LayergroupController.prototype.register = function(app) { this.tile.bind(this)); app.get(app.base_url_mapconfig + - '/:token/:z/:x/:y.:format', cors(), userMiddleware, + '/:token/:z/:x/:y.(:format)', cors(), userMiddleware, this.tile.bind(this)); app.get(app.base_url_mapconfig + From 8426dd00f1ca951c7e3c3f5d157435a5ae1eb1e7 Mon Sep 17 00:00:00 2001 From: csobier Date: Tue, 2 May 2017 12:48:55 -0400 Subject: [PATCH 063/402] added print attributions from Docs FAQs to Static Maps API content --- docs/static_maps_api.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/static_maps_api.md b/docs/static_maps_api.md index 736d90e0..91db2a6d 100644 --- a/docs/static_maps_api.md +++ b/docs/static_maps_api.md @@ -150,6 +150,10 @@ It is important to note that generated images are cached from the live data refe * Image resolution is set to 72 DPI * JPEG quality is 85% * Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries. +* If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code: + +{% highlight javascript %}attribution: '© OpenStreetMap contributors, © CARTO +{% endhighlight %} ## Examples From e97466378ee9ccc57853b208ae978a8bba59c3de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 3 May 2017 11:17:51 +0200 Subject: [PATCH 064/402] Add test for different formats to anonymous maps and named maps with placeholders --- test/acceptance/buffer-size-format.js | 138 ++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 9 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 8c8f6f7b..f573057c 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -99,6 +99,50 @@ describe('buffer size per format', function () { assert.equal(features.length, 9); callback(); } + }, + { + desc: 'should get geojson tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'geojson', + fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson', + mapConfig: createMapConfig({ geojson: 0 }), + assert: function (tile, callback) { + var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); + assert.equal(tile.features.length, dataFixture.features.length); + callback(); + } + }, + { + desc: 'should get geojson tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'geojson', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson', + mapConfig: createMapConfig({ geojson: 128 }), + assert: function (tile, callback) { + var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); + assert.equal(tile.features.length, dataFixture.features.length); + callback(); + } + }, + { + desc: 'should get grid.json tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', + mapConfig: createMapConfig({ 'grid.json': 0 }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback); + } + }, + { + desc: 'should get grid.json tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', + mapConfig: createMapConfig({ 'grid.json': 128 }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); + } } ]; @@ -193,6 +237,82 @@ describe('buffer size per format for named maps', function () { assert: function (tile, callback) { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } + }, + { + desc: 'should get geojson tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'geojson', + placeholders: { buffersize_geojson: 0 }, + fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson', + template: createBufferSizeTemplate('named-default-buffer-size-by-format-geojson', { + geojson: '<%= buffersize_geojson %>' + }, { + "buffersize_geojson": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); + assert.equal(tile.features.length, dataFixture.features.length); + callback(); + } + }, + { + desc: 'should get geojson tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'geojson', + placeholders: { buffersize_geojson: 128 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson', + template: createBufferSizeTemplate('named-custom-buffer-size-by-format-geojson', { + geojson: '<%= buffersize_geojson %>' + }, { + "buffersize_geojson": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); + assert.equal(tile.features.length, dataFixture.features.length); + callback(); + } + }, + { + desc: 'should get grid.json tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + placeholders: { buffersize_gridjson: 0 }, + fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', + template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', { + 'grid.json': '<%= buffersize_gridjson %>' + }, { + "buffersize_gridjson": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback); + } + }, + { + desc: 'should get grid.json tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + placeholders: { buffersize_gridjson: 128 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', + template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', { + 'grid.json': '<%= buffersize_gridjson %>' + }, { + "buffersize_gridjson": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); + } } ]; @@ -224,7 +344,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get png tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'png', - placeholders: { + placeholders: { buffersize: { png: 0 } @@ -239,7 +359,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get png tile using buffer-size 128 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'png', - placeholders: { + placeholders: { buffersize: { png: 128 } @@ -254,7 +374,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get mvt tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'mvt', - placeholders: { + placeholders: { buffersize: { mvt: 0 } @@ -279,7 +399,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get mvt tile using buffer-size 128 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'mvt', - placeholders: { + placeholders: { buffersize: { mvt: 128 } @@ -304,7 +424,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get geojson tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'geojson', - placeholders: { + placeholders: { buffersize: { geojson: 0 } @@ -321,7 +441,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get geojson tile using buffer-size 128 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'geojson', - placeholders: { + placeholders: { buffersize: { geojson: 128 } @@ -338,7 +458,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get grid.json tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', - placeholders: { + placeholders: { buffersize: { 'grid.json': 0 } @@ -353,7 +473,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get grid.json tile using buffer-size 128 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', - placeholders: { + placeholders: { buffersize: { 'grid.json': 128 } @@ -363,7 +483,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { assert: function (tile, callback) { assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); } - }, + } ]; testCases.forEach(function (test) { From 05d3b3bf669ecc9e73fa40e29285fb2a6da4175a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 3 May 2017 17:21:08 +0200 Subject: [PATCH 065/402] Point windshaft to specific commit for buffer-size configuration --- package.json | 2 +- yarn.lock | 106 +++++++++++++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index 07f158b7..f91ac976 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "3.1.1", + "windshaft": "cartodb/windshaft#548-vector-buffer-size", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 1386aaaa..1addd775 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -22,8 +22,8 @@ accepts@~1.2.12: negotiator "0.5.3" ajv@^4.9.1: - version "4.11.7" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.7.tgz#8655a5d86d0824985cc471a1d913fb6729a0ec48" + version "4.11.8" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" dependencies: co "^4.6.0" json-stable-stringify "^1.0.1" @@ -205,7 +205,7 @@ camshaft@0.54.1: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,7 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -239,6 +231,14 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -612,8 +612,8 @@ express@~4.13.3: vary "~1.0.1" extend@~3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4" + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" extsprintf@1.0.2: version "1.0.2" @@ -716,7 +716,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3: +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -728,21 +728,17 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" getpass@^0.1.1: - version "0.1.6" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.6.tgz#283ffd9fc1256840875311c1b60e8c40187110e6" + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" dependencies: assert-plus "^1.0.0" -glob@3.2.3, "glob@~ 3.2.1": +glob@3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -781,6 +777,13 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" +"glob@~ 3.2.1": + version "3.2.11" + resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" + dependencies: + inherits "2" + minimatch "0.3" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -812,8 +815,8 @@ growl@1.8.1: resolved "https://registry.yarnpkg.com/growl/-/growl-1.8.1.tgz#4b2dec8d907e93db336624dcec0183502f8c9428" handlebars@^4.0.1: - version "4.0.6" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.6.tgz#2ce4484850537f9c97a8026d5399b935c4ed4ed7" + version "4.0.8" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -939,7 +942,7 @@ is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" -is-buffer@^1.0.2: +is-buffer@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc" @@ -1084,10 +1087,10 @@ jsprim@^1.2.2: verror "1.3.6" kind-of@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.1.0.tgz#475d698a5e49ff5e53d14e3e732429dc8bf4cf47" + version "3.2.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.0.tgz#b58abe4d5c044ad33726a8c1525b48cf891bff07" dependencies: - is-buffer "^1.0.2" + is-buffer "^1.1.5" lazy-cache@^1.0.3: version "1.0.4" @@ -1224,6 +1227,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" +minimatch@0.3: + version "0.3.0" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" + dependencies: + lru-cache "2" + sigmund "~1.0.0" + minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -1393,14 +1403,10 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0: +object-assign@4.1.0, object-assign@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1517,7 +1523,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +"pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1828,14 +1834,18 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +"semver@2 || 3 || 4 || 5", semver@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" +semver@^5.1.0, semver@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + semver@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540" @@ -1844,10 +1854,6 @@ semver@~4.3.3: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" -semver@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" - send@0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" @@ -2098,17 +2104,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb1": +tilelive-bridge@cartodb/tilelive-bridge#buffer-size: version "2.3.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/9df4bde07f3cf9ffbda42657066559e2b8875e1b" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb1": +tilelive-mapnik@cartodb/tilelive-mapnik#fix-buffer-size-0: version "0.6.18-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/c67400573572e836609dd75a7acec8f414a50dbe" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2269,9 +2275,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.1.1: +windshaft@cartodb/windshaft#548-vector-buffer-size: version "3.1.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.1.tgz#5e1b20fa9bcc6f4e04f67738b18a35cb19cadbd1" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2288,8 +2294,8 @@ windshaft@3.1.1: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb1 - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb1 + tilelive-bridge cartodb/tilelive-bridge#buffer-size + tilelive-mapnik cartodb/tilelive-mapnik#fix-buffer-size-0 torque.js "~2.11.0" underscore "~1.6.0" From 55a351d7514051d1bb53008485b9f13c267c4aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 3 May 2017 17:35:55 +0200 Subject: [PATCH 066/402] Point windshaft to a specific buffer-size commit --- package.json | 2 +- yarn.lock | 182 ++++++++++++++++++++++++++++----------------------- 2 files changed, 102 insertions(+), 82 deletions(-) diff --git a/package.json b/package.json index 9a7c5dd0..af824cc4 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "3.1.0", + "windshaft": "cartodb/windshaft#548-vector-buffer-size", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 9a3c48fe..79f89643 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22,8 +22,8 @@ accepts@~1.2.12: negotiator "0.5.3" ajv@^4.9.1: - version "4.11.5" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.5.tgz#b6ee74657b993a01dce44b7944d56f485828d5bd" + version "4.11.8" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" dependencies: co "^4.6.0" json-stable-stringify "^1.0.1" @@ -57,11 +57,11 @@ aproba@^1.0.3: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" are-we-there-yet@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz#80e470e95a084794fe1899262c5667c6e88de1b3" + version "1.1.4" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.4.tgz#bb5dca382bb94f05e15194373d16fd3ba1ca110d" dependencies: delegates "^1.0.0" - readable-stream "^2.0.0 || ^1.1.13" + readable-stream "^2.0.6" argparse@^1.0.7: version "1.0.9" @@ -151,13 +151,13 @@ boom@2.x.x: hoek "2.x.x" brace-expansion@^1.0.0: - version "1.1.6" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.6.tgz#7197d7eaa9b87e648390ea61fc66c84427420df9" + version "1.1.7" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" dependencies: balanced-match "^0.4.1" concat-map "0.0.1" -buffer-shims@^1.0.0: +buffer-shims@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" @@ -223,14 +223,6 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -239,6 +231,14 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -612,8 +612,8 @@ express@~4.13.3: vary "~1.0.1" extend@~3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4" + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" extsprintf@1.0.2: version "1.0.2" @@ -650,8 +650,8 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" form-data@~2.1.1: - version "2.1.2" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.2.tgz#89c3534008b97eada4cbb157d58f6f5df025eae4" + version "2.1.4" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1" dependencies: asynckit "^0.4.0" combined-stream "^1.0.5" @@ -687,8 +687,8 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: rimraf "2" gauge@~2.7.1: - version "2.7.3" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.3.tgz#1c23855f962f17b3ad3d0dc7443f304542edfe09" + version "2.7.4" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" dependencies: aproba "^1.0.3" console-control-strings "^1.0.0" @@ -716,9 +716,9 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.2.tgz#886bc5bf0beb7db96e81bcbba078818de5a62683" +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.3" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" generic-pool@~2.1.1: version "2.1.1" @@ -728,21 +728,17 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" getpass@^0.1.1: - version "0.1.6" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.6.tgz#283ffd9fc1256840875311c1b60e8c40187110e6" + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" dependencies: assert-plus "^1.0.0" -glob@3.2.3, "glob@~ 3.2.1": +glob@3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -781,6 +777,13 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" +"glob@~ 3.2.1": + version "3.2.11" + resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" + dependencies: + inherits "2" + minimatch "0.3" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -794,8 +797,8 @@ graceful-fs@~2.0.0: resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" grainstore@~1.6.0: - version "1.6.2" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.2.tgz#f5d098a8da607f23db08f3263894c8f234a8a59e" + version "1.6.3" + resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.3.tgz#6900cc811aadc1ed2c00fcd429c672f8b8e1a5cb" dependencies: carto "0.16.3" debug "~2.2.0" @@ -812,8 +815,8 @@ growl@1.8.1: resolved "https://registry.yarnpkg.com/growl/-/growl-1.8.1.tgz#4b2dec8d907e93db336624dcec0183502f8c9428" handlebars@^4.0.1: - version "4.0.6" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.6.tgz#2ce4484850537f9c97a8026d5399b935c4ed4ed7" + version "4.0.8" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -876,8 +879,8 @@ hoek@2.x.x: resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" hosted-git-info@^2.1.4: - version "2.4.1" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.1.tgz#4b0445e41c004a8bd1337773a4ff790ca40318c8" + version "2.4.2" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.2.tgz#0076b9f46a270506ddbaaea56496897460612a67" htmlparser2@3.8.x: version "3.8.3" @@ -939,7 +942,7 @@ is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" -is-buffer@^1.0.2: +is-buffer@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc" @@ -1029,8 +1032,8 @@ js-base64@^2.1.9: resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" js-yaml@3.x, js-yaml@^3.4.6: - version "3.8.2" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.2.tgz#02d3e2c0f6beab20248d412c352203827d786721" + version "3.8.3" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.3.tgz#33a05ec481c850c8875929166fe1beb61c728766" dependencies: argparse "^1.0.7" esprima "^3.1.1" @@ -1084,10 +1087,10 @@ jsprim@^1.2.2: verror "1.3.6" kind-of@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.1.0.tgz#475d698a5e49ff5e53d14e3e732429dc8bf4cf47" + version "3.2.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.0.tgz#b58abe4d5c044ad33726a8c1525b48cf891bff07" dependencies: - is-buffer "^1.0.2" + is-buffer "^1.1.5" lazy-cache@^1.0.3: version "1.0.4" @@ -1210,7 +1213,7 @@ mime-db@~1.27.0: version "1.27.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1" -mime-types@^2.1.12, mime-types@~2.1.13, mime-types@~2.1.6, mime-types@~2.1.7: +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: version "2.1.15" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed" dependencies: @@ -1224,6 +1227,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" +minimatch@0.3: + version "0.3.0" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" + dependencies: + lru-cache "2" + sigmund "~1.0.0" + minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -1305,14 +1315,18 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.5.0: - version "2.5.1" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" nan@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" +nan@~2.5.0: + version "2.5.1" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" + ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" @@ -1364,8 +1378,8 @@ nopt@^4.0.1: osenv "^0.1.4" normalize-package-data@^2.3.2: - version "2.3.6" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.6.tgz#498fa420c96401f787402ba21e600def9f981fff" + version "2.3.8" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.8.tgz#d819eda2a9dedbd1ffa563ea4071d936782295bb" dependencies: hosted-git-info "^2.1.4" is-builtin-module "^1.0.0" @@ -1493,10 +1507,10 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.6.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.6.0.tgz#2e300199927b6d7db6be71e2e3435dddddf07b41" + version "1.7.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" dependencies: - generic-pool "2.4.2" + generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: @@ -1509,7 +1523,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +"pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1566,8 +1580,8 @@ postcss@5.0.19: supports-color "^3.1.2" postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8: - version "5.2.16" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.16.tgz#732b3100000f9ff8379a48a53839ed097376ad57" + version "5.2.17" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.17.tgz#cf4f597b864d65c8a492b2eabe9d706c879c388b" dependencies: chalk "^1.1.3" js-base64 "^2.1.9" @@ -1660,8 +1674,8 @@ raw-body@~2.1.5: unpipe "1.0.0" rc@^1.1.7: - version "1.1.7" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.1.7.tgz#c5ea564bb07aff9fd3a5b32e906c1d3a65940fea" + version "1.2.1" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.1.tgz#2e03e8e42ee450b8cb3dce65be1bf8974e1dfd95" dependencies: deep-extend "~0.4.0" ini "~1.3.0" @@ -1692,16 +1706,16 @@ readable-stream@1.1, readable-stream@~1.1.9: isarray "0.0.1" string_decoder "~0.10.x" -"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.1.4: - version "2.2.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.6.tgz#8b43aed76e71483938d12a8d46c6cf1a00b1f816" +readable-stream@^2.0.6, readable-stream@^2.1.4: + version "2.2.9" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" dependencies: - buffer-shims "^1.0.0" + buffer-shims "~1.0.0" core-util-is "~1.0.0" inherits "~2.0.1" isarray "~1.0.0" process-nextick-args "~1.0.6" - string_decoder "~0.10.x" + string_decoder "~1.0.0" util-deprecate "~1.0.1" readable-stream@~1.0.2: @@ -1730,7 +1744,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1755,7 +1769,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1970,8 +1984,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.11.0.tgz#2d8d5ebb4a6fab28ffba37fa62a90f4a3ea59d77" + version "1.13.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -2018,6 +2032,12 @@ string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" +string_decoder@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" + dependencies: + buffer-shims "~1.0.0" + stringstream@~0.0.4: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2084,17 +2104,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: +tilelive-bridge@cartodb/tilelive-bridge#buffer-size: version "2.3.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/9df4bde07f3cf9ffbda42657066559e2b8875e1b" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb1: +tilelive-mapnik@cartodb/tilelive-mapnik#fix-buffer-size-0: version "0.6.18-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/c67400573572e836609dd75a7acec8f414a50dbe" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2163,15 +2183,15 @@ type-detect@^1.0.0: resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" type-is@~1.6.10, type-is@~1.6.6: - version "1.6.14" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.14.tgz#e219639c17ded1ca0789092dd54a03826b817cb2" + version "1.6.15" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410" dependencies: media-typer "0.3.0" - mime-types "~2.1.13" + mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.18" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.18.tgz#925d14bae48ab62d1883b41afe6e2261662adb8e" + version "2.8.22" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.22.tgz#d54934778a8da14903fa29a326fb24c0ab51a1a0" dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -2255,9 +2275,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.0.tgz#dfac2dd27a2db97e35231510743ad7db574da18f" +windshaft@cartodb/windshaft#548-vector-buffer-size: + version "3.1.1" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2274,8 +2294,8 @@ windshaft@3.1.0: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb1 - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb1 + tilelive-bridge cartodb/tilelive-bridge#buffer-size + tilelive-mapnik cartodb/tilelive-mapnik#fix-buffer-size-0 torque.js "~2.11.0" underscore "~1.6.0" From 6bd753746794fa52845325e49205a8a26739a1dc Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 5 May 2017 15:57:02 +0200 Subject: [PATCH 067/402] Upgrade deps --- NEWS.md | 3 +++ package.json | 6 +++--- yarn.lock | 22 +++++++++++----------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/NEWS.md b/NEWS.md index 70ee13d9..fc24bd0f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,9 @@ ## 3.6.4 Released 2017-mm-dd + - Upgrade cartodb-psql to [0.8.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.8.0). + - Upgrades camshaft to [0.54.2](https://github.com/CartoDB/camshaft/releases/tag/0.54.2). + - Upgrades windshaft to [3.1.2](https://github.com/CartoDB/windshaft/releases/tag/3.1.2). ## 3.6.3 diff --git a/package.json b/package.json index 07f158b7..ed35366d 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.54.1", - "cartodb-psql": "~0.7.1", + "camshaft": "0.54.2", + "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", "debug": "~2.2.0", @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "3.1.1", + "windshaft": "3.1.2", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 1386aaaa..65417812 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,13 +194,13 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.54.1: - version "0.54.1" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.1.tgz#0eea25ee9e5a1cf32b0f59fef48f8f3590ebfe17" +camshaft@0.54.2: + version "0.54.2" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.2.tgz#eb9f2b8b6e948aed492d206bc32ff774dcfd29c2" dependencies: async "^1.5.2" bunyan "1.8.1" - cartodb-psql "0.7.1" + cartodb-psql "0.8.0" debug "^2.2.0" dot "^1.0.3" request "^2.69.0" @@ -245,9 +245,9 @@ cartocolor@4.0.0: dependencies: colorbrewer "1.0.0" -cartodb-psql@0.7.1, cartodb-psql@~0.7.1: - version "0.7.1" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.7.1.tgz#578ce04db9262f1296845dec643461105a594e22" +cartodb-psql@0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" dependencies: debug "~2.2.0" pg cartodb/node-postgres#6.1.2-cdb1 @@ -2269,14 +2269,14 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.1.tgz#5e1b20fa9bcc6f4e04f67738b18a35cb19cadbd1" +windshaft@3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.2.tgz#78c257bafe63c41df8cd7e4aa7ea6ea72ed1a5bd" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.7.1" + cartodb-psql "0.8.0" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" From 4e6e267f100ccf60db3e51c150076a176a3af811 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 5 May 2017 16:03:38 +0200 Subject: [PATCH 068/402] Fix news --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index fc24bd0f..2cfd508c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,8 @@ ## 3.6.4 Released 2017-mm-dd + +Announcements: - Upgrade cartodb-psql to [0.8.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.8.0). - Upgrades camshaft to [0.54.2](https://github.com/CartoDB/camshaft/releases/tag/0.54.2). - Upgrades windshaft to [3.1.2](https://github.com/CartoDB/windshaft/releases/tag/3.1.2). From 36e9239056511daa86d313e1832105a488724507 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 5 May 2017 16:03:53 +0200 Subject: [PATCH 069/402] Release 3.6.4 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2cfd508c..5257a58a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.6.4 -Released 2017-mm-dd +Released 2017-05-05 Announcements: - Upgrade cartodb-psql to [0.8.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.8.0). From 0ac53db73a976e7c88a0abe03881eda1577cdff8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 5 May 2017 16:04:25 +0200 Subject: [PATCH 070/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5257a58a..507e62f4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.6.5 +Released 2017-mm-dd + + ## 3.6.4 Released 2017-05-05 diff --git a/package.json b/package.json index ed35366d..1dc040a4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.4", + "version": "3.6.5", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 3f6c8fa51c032232b97ab826ce76f960918b4989 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 8 May 2017 18:42:40 +0200 Subject: [PATCH 071/402] Use windshaft-stats to get layer stats --- .gitignore | 1 + lib/cartodb/controllers/map.js | 49 ++++++++++++++++--- .../provider/dummy-mapconfig-provider.js | 20 ++++++++ package.json | 1 + 4 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js diff --git a/.gitignore b/.gitignore index b39f2801..d49bbc6e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ redis.pid test.log npm-debug.log coverage/ +.DS_Store diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b3ca2b4a..0cdf75f1 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var step = require('step'); var windshaft = require('windshaft'); +var windshaftStats = require('windshaft-stats'); var QueryTables = require('cartodb-query-tables'); var ResourceLocator = require('../models/resource-locator'); @@ -19,6 +20,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); +var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconfig-provider'); /** * @param {AuthApi} authApi @@ -33,7 +35,8 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) { + surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, + rendererCache) { BaseController.call(this, authApi, pgConnection); @@ -47,6 +50,8 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.mapConfigAdapter = mapConfigAdapter; this.resourceLocator = new ResourceLocator(global.environment); + + this.rendererCache = rendererCache; } util.inherits(MapController, BaseController); @@ -161,7 +166,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, req.params, this); }, function finish(err, layergroup) { if (err) { @@ -214,6 +219,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn var mapConfigProvider; var mapConfig; + var rendererParams; step( function setupParams(){ @@ -238,8 +244,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); mapConfigProvider.getMapConfig(this); }, - function createLayergroup(err, mapConfig_, rendererParams) { + function createLayergroup(err, mapConfig_, _rendererParams) { assert.ifError(err); + rendererParams = _rendererParams; mapConfig = mapConfig_; self.mapBackend.createLayergroup( mapConfig, rendererParams, @@ -249,7 +256,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, rendererParams, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -272,7 +279,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); }; -MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) { +MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { var self = this; var username = req.context.user; @@ -318,6 +325,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la var dbName = req.params.dbname; var layergroupId = layergroup.layergroupid; + var dbConnection; step( function getPgConnection() { @@ -325,7 +333,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la }, function getAffectedTablesAndLastUpdatedTime(err, connection) { assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this); + dbConnection = connection; + QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { req.profiler.done('queryTablesAndLastUpdated'); @@ -352,6 +361,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la return null; }, + function fetchLayersMetadata(err) { + assert.ifError(err); + var next = this; + var mapConfigProvider = new DummyMapConfigProvider(mapconfig, rendererParams); + var layerStats = windshaftStats(); + layerStats.getStats(self.rendererCache, rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { + if (err) { + return next(err); + } + if (layersStats) { + res.metadata = res.metadata || {}; + res.metadata.layers = layersStats; + + // backwards compatibility for torque + var torqueMetadata = layersStats.reduce(function(acc, layer, layerId) { + if (layer.type === 'torque') { + acc[layerId] = layer.meta; + } + return acc; + }, {}); + if (Object.keys(torqueMetadata).length) { + res.metadata.torque = torqueMetadata; + } + } + + return next(); + }); + }, function finish(err) { done(err); } diff --git a/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js b/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js new file mode 100644 index 00000000..0b8942c5 --- /dev/null +++ b/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js @@ -0,0 +1,20 @@ +var util = require('util'); +var MapStoreMapConfigProvider = require('windshaft').model.provider.MapStoreMapConfig; + +function DummyMapConfigProvider(mapConfig, params) { + MapStoreMapConfigProvider.call(this, undefined, params); + + this.mapConfig = mapConfig; +} + +util.inherits(DummyMapConfigProvider, MapStoreMapConfigProvider); + +module.exports = DummyMapConfigProvider; + +DummyMapConfigProvider.prototype.setParams = function(params) { + this.params = params; +}; + +DummyMapConfigProvider.prototype.getMapConfig = function(callback) { + return callback(null, this.mapConfig, this.params, {}); +}; diff --git a/package.json b/package.json index f91ac976..9bbb5125 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "turbo-carto": "0.19.0", "underscore": "~1.6.0", "windshaft": "cartodb/windshaft#548-vector-buffer-size", + "windshaft-stats": "0.0.1", "yargs": "~5.0.0" }, "devDependencies": { From 525c0f2afaa99cae395e8e12bd28f47f55e8c236 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 11:58:48 +0200 Subject: [PATCH 072/402] Added rendererCache as dependency for map controller --- lib/cartodb/controllers/map.js | 1 + lib/cartodb/server.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 0cdf75f1..b8242b59 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -32,6 +32,7 @@ var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconf * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables * @param {MapConfigAdapter} mapConfigAdapter + * @param {RendererCache} rendererCache * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index b695a367..e14c0389 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -207,7 +207,8 @@ module.exports = function(serverOptions) { surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache, - mapConfigAdapter + mapConfigAdapter, + rendererCache ).register(app); new controller.NamedMaps( From 568d6b5458e2803c5179a86e5df43402f46924c8 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 11:59:11 +0200 Subject: [PATCH 073/402] Include the layers metadata into the layergroup object --- lib/cartodb/controllers/map.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b8242b59..4bcc4911 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -372,8 +372,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la return next(err); } if (layersStats) { - res.metadata = res.metadata || {}; - res.metadata.layers = layersStats; + layergroup.metadata = layergroup.metadata || {}; + layergroup.metadata.layers = layersStats; // backwards compatibility for torque var torqueMetadata = layersStats.reduce(function(acc, layer, layerId) { @@ -383,7 +383,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la return acc; }, {}); if (Object.keys(torqueMetadata).length) { - res.metadata.torque = torqueMetadata; + layergroup.metadata.torque = torqueMetadata; } } From 15c68711aa44bac74027a21f883259f5152b9ea3 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 12:31:16 +0200 Subject: [PATCH 074/402] Create stats backend to decouple stats logic from map controller --- lib/cartodb/backends/stats.js | 12 ++++++++++++ lib/cartodb/controllers/map.js | 10 ++++------ lib/cartodb/server.js | 6 +++++- 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 lib/cartodb/backends/stats.js diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js new file mode 100644 index 00000000..2179db1b --- /dev/null +++ b/lib/cartodb/backends/stats.js @@ -0,0 +1,12 @@ +var windshaftStats = require('windshaft-stats'); + +function StatsBackend(rendererCache) { + this.rendererCache = rendererCache; +}; + +module.exports = StatsBackend; + +StatsBackend.prototype.getStats = function(params, dbConnection, mapConfigProvider, callback) { + var layerStats = windshaftStats(); + layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); +}; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4bcc4911..f3211190 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,7 +2,6 @@ var _ = require('underscore'); var assert = require('assert'); var step = require('step'); var windshaft = require('windshaft'); -var windshaftStats = require('windshaft-stats'); var QueryTables = require('cartodb-query-tables'); var ResourceLocator = require('../models/resource-locator'); @@ -32,12 +31,12 @@ var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconf * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables * @param {MapConfigAdapter} mapConfigAdapter - * @param {RendererCache} rendererCache + * @param {StatsBackend} statsBackend * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, - rendererCache) { + statsBackend) { BaseController.call(this, authApi, pgConnection); @@ -52,7 +51,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.mapConfigAdapter = mapConfigAdapter; this.resourceLocator = new ResourceLocator(global.environment); - this.rendererCache = rendererCache; + this.statsBackend = statsBackend; } util.inherits(MapController, BaseController); @@ -366,8 +365,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la assert.ifError(err); var next = this; var mapConfigProvider = new DummyMapConfigProvider(mapconfig, rendererParams); - var layerStats = windshaftStats(); - layerStats.getStats(self.rendererCache, rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { + self.statsBackend.getStats(rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { if (err) { return next(err); } diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index e14c0389..5ce31f13 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -41,6 +41,8 @@ var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter' var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter'); var MapConfigAdapter = require('./models/mapconfig/adapter'); +var StatsBackend = require('./backends/stats'); + module.exports = function(serverOptions) { // Make stats client globally accessible global.statsClient = StatsClient.getInstance(serverOptions.statsd); @@ -150,6 +152,8 @@ module.exports = function(serverOptions) { var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); + var statsBackend = new StatsBackend(rendererCache); + var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; @@ -208,7 +212,7 @@ module.exports = function(serverOptions) { userLimitsApi, layergroupAffectedTablesCache, mapConfigAdapter, - rendererCache + statsBackend ).register(app); new controller.NamedMaps( From 759d28f12f40854f5bcb8ce44c92f2c8e9343265 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 12:51:22 +0200 Subject: [PATCH 075/402] Include feature flag to enable/disable stats --- lib/cartodb/backends/stats.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 2179db1b..d2e989e3 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -7,6 +7,14 @@ function StatsBackend(rendererCache) { module.exports = StatsBackend; StatsBackend.prototype.getStats = function(params, dbConnection, mapConfigProvider, callback) { - var layerStats = windshaftStats(); - layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + var enabledFeatures = global.environment.enabledFeatures; + var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; + var layerStats; + if (layerMetadataEnabled) { + layerStats = windshaftStats(); + layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + } else { + layerStats = windshaftStats('torque'); + layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + } }; From b25540720ca2e5e2b8dd3f285354b0b7ff522a6a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 14:36:35 +0200 Subject: [PATCH 076/402] Added acceptance tests for stats functionallity --- .../stats/mapnik_stats_layergroup.js | 260 ++++++++++++++++++ test/acceptance/stats/multilayer_stats.js | 219 +++++++++++++++ test/support/sql/windshaft.test.sql | 2 + 3 files changed, 481 insertions(+) create mode 100644 test/acceptance/stats/mapnik_stats_layergroup.js create mode 100644 test/acceptance/stats/multilayer_stats.js diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js new file mode 100644 index 00000000..746f6c5c --- /dev/null +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -0,0 +1,260 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +describe('Create mapnik layergroup', function() { + before(function() { + this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; + global.environment.enabledFeatures.layerMetadata = true; + }); + + after(function() { + global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; + }); + + var cartocssVersion = '2.3.0'; + var cartocss = '#layer { line-width:16; }'; + + var mapnikLayer1 = { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer2 = { + type: 'mapnik', + options: { + sql: 'select * from test_table_2 limit 2', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer3 = { + type: 'mapnik', + options: { + sql: 'select * from test_table_3 limit 2', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer4 = { + type: 'mapnik', + options: { + sql: [ + 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address', + ' from test_table t1, test_table_2 t2', + ' where t1.cartodb_id = t2.cartodb_id;' + ].join(''), + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var httpLayer = { + type: 'http', + options: { + urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png', + subdomains: ['a','b','c'] + } + }; + + var mapnikLayerGeomColumn = { + type: 'mapnik', + options: { + sql: 'select *, the_geom as my_geom from test_table_3 limit 2', + geom_column: 'my_geom', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + function mapnikBasicLayerId(index) { + return 'layer' + index; + } + function typeLayerId(type, index) { + return type + '-' + mapnikBasicLayerId(index); + } + + it('with one mapnik layer should response with meta-stats for that layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + mapnikLayer2 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + testClient.drain(done); + }); + }); + + it('with three mapnik layer should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + mapnikLayer2, + mapnikLayer3 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); + assert.equal(layergroup.metadata.layers[2].meta.stats[0].features, 5); + testClient.drain(done); + }); + }); + + it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer4, + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer3, + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); + assert.ok(!layergroup.metadata.layers[2]); + testClient.drain(done); + }); + }); + + it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + httpLayer + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[1].type, 'http'); + testClient.drain(done); + }); + }); + + it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + httpLayer, + mapnikLayer1 + ] + }); + + testClient.getLayergroup(function (err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[0].type, 'http'); + assert.ok(!layergroup.metadata.layers[0].meta.cartocss); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); + assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); + testClient.drain(done); + }); + }); + + it('should work with different geom_column', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayerGeomColumn + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + // we don't care about stats here as is an aliased column + assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('features')); + testClient.drain(done); + }); + }); +}); diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js new file mode 100644 index 00000000..4571bf6a --- /dev/null +++ b/test/acceptance/stats/multilayer_stats.js @@ -0,0 +1,219 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +describe('multilayer stats disabled', function() { + + before(function () { + global.environment.enabledFeatures.layerMetadata = false; + }); + + after(function () { + global.environment.enabledFeatures.layerMetadata = true; + }); + + + function testLayerMetadataStats(testScenario) { + + it(testScenario.desc, function(done) { + var mapConfig = { + version: '1.3.0', + layers: testScenario.layers + }; + + var testClient = new TestClient(mapConfig); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + layergroup.metadata.layers.forEach(function (layer) { + if (layer.type !== 'torque' && layer.type !== 'mapnik') { + assert.ok('stats' in layer.meta); + } else if (layer.type !== 'torque') { + assert.ok('stats' in layer.meta); + assert.ok('cartocss' in layer.meta); + } else { + assert.ok('cartocss' in layer.meta); + // check torque metadata at least match in number + var torqueLayers = mapConfig.layers.filter(function(layer) { return layer.type === 'torque'; }); + if (torqueLayers.length) { + assert.equal(Object.keys(layergroup.metadata.torque).length, torqueLayers.length); + } + } + }); + + testClient.drain(done); + }); + }); + } + + var cartocssVersion = '2.3.0'; + var cartocss = '#layer { line-width:16; }'; + var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + + var httpLayer = { + type: 'http', + options: { + urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png', + subdomains: ['a','b','c'] + } + }; + + var torqueLayer = { + type: 'torque', + options: { + sql: "select 1 id, '1970-01-02'::date d, 'POINT(0 0)'::geometry the_geom_webmercator", + cartocss: [ + "Map {", + "-torque-frame-count:2;", + "-torque-resolution:3;", + "-torque-time-attribute:d;", + "-torque-aggregation-function:'count(id)';", + "}" + ].join(' '), + cartocss_version: '2.0.1' + } + }; + + var mapnikLayer = { + type: 'mapnik', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikInteractivityLayer = { + type: 'mapnik', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var cartodbLayer = { + type: 'cartodb', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var cartodbInteractivityLayer = { + type: 'cartodb', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var cartodbWadusInteractivityLayer = { + type: 'cartodb', + options: { + sql: sqlWadus, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'wadus' + } + }; + + var noTypeLayer = { + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var noTypeInteractivityLayer = { + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var testScenarios = [ + { + desc: 'one layer, no interactivity', + layers: [cartodbLayer] + }, + { + desc: 'two layers, different interactivity columns', + layers: [ + cartodbWadusInteractivityLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'torque + interactivity layers', + layers: [ + torqueLayer, + cartodbWadusInteractivityLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'interactivity + torque + interactivity', + layers: [ + cartodbInteractivityLayer, + torqueLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'http + interactivity + torque + no interactivity + torque + interactivity', + layers: [ + httpLayer, + cartodbInteractivityLayer, + torqueLayer, + cartodbLayer, + torqueLayer, + cartodbWadusInteractivityLayer + ] + }, + { + desc: 'mapnik type – two layers, interactivity mix', + layers: [ + mapnikLayer, + mapnikInteractivityLayer + ] + }, + { + desc: 'mapnik type – http + interactivity + torque + interactivity', + layers: [ + httpLayer, + mapnikInteractivityLayer, + torqueLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'no type – two layers, interactivity mix', + layers: [ + noTypeLayer, + noTypeInteractivityLayer + ] + }, + { + desc: 'no type – http + interactivity + torque + interactivity', + layers: [ + httpLayer, + noTypeInteractivityLayer, + torqueLayer, + noTypeInteractivityLayer + ] + } + ]; + + testScenarios.forEach(testLayerMetadataStats); + +}); diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index eb495028..91484950 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -649,3 +649,5 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters int END; $$ LANGUAGE plpgsql; GRANT ALL ON FUNCTION cdb_crankshaft.CDB_KMeans(text, integer, integer) TO :TESTUSER; + +ANALYZE; From 474d68687c60e71633d5a7b81a00d3aed37bb7cd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 14:47:39 +0200 Subject: [PATCH 077/402] Include vertexCount in the stats tests --- .../stats/mapnik_stats_layergroup.js | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 746f6c5c..78d873af 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -92,7 +92,8 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -109,9 +110,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -129,11 +132,14 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -149,8 +155,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -167,11 +175,15 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -188,11 +200,14 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); @@ -212,7 +227,8 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -233,7 +249,8 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -253,7 +270,8 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('features')); + assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); + assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('vertexCount')); testClient.drain(done); }); }); From 7bb039b13c218d441da76cb3635bf8780b5236a9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 9 May 2017 14:58:25 +0200 Subject: [PATCH 078/402] Upgrades camshaft to 0.54.3 --- NEWS.md | 3 +++ package.json | 2 +- yarn.lock | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index 507e62f4..6c59d958 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.6.5 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.54.3](https://github.com/CartoDB/camshaft/releases/tag/0.54.3). + ## 3.6.4 Released 2017-05-05 diff --git a/package.json b/package.json index 1dc040a4..56e6fd04 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.54.2", + "camshaft": "0.54.3", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 65417812..fb7b4d06 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.54.2: - version "0.54.2" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.2.tgz#eb9f2b8b6e948aed492d206bc32ff774dcfd29c2" +camshaft@0.54.3: + version "0.54.3" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.3.tgz#ba0b0fc00c5f82dc5d6b7d7649ed3412f6a9902b" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -205,7 +205,7 @@ camshaft@0.54.2: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1738,7 +1738,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1763,7 +1763,7 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.81.0: +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -2098,7 +2098,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb1": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: version "2.3.1-cdb1" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" dependencies: @@ -2106,7 +2106,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb1": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb1: version "0.6.18-cdb1" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" dependencies: From 5b96db2ba2e822b997362c6b0941b9df96590bfa Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 9 May 2017 15:06:00 +0200 Subject: [PATCH 079/402] Release 3.6.5 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6c59d958..6a45c834 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.6.5 -Released 2017-mm-dd +Released 2017-05-09 Announcements: - Upgrades camshaft to [0.54.3](https://github.com/CartoDB/camshaft/releases/tag/0.54.3). From 1a197bb9cfd390e56429a580019b5eb6cd6aaea3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 9 May 2017 15:06:49 +0200 Subject: [PATCH 080/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6a45c834..a4ef9bdd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.6.6 +Released 2017-mm-dd + + ## 3.6.5 Released 2017-05-09 diff --git a/package.json b/package.json index 56e6fd04..e7a6993e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.5", + "version": "3.6.6", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From b87e4428018de06f1516f4a9862c526a366c7fdf Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 18:24:11 +0200 Subject: [PATCH 081/402] Remove vertex count from stats tests --- .../stats/mapnik_stats_layergroup.js | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 78d873af..4a327a01 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -93,7 +93,6 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -111,10 +110,8 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -133,13 +130,10 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -156,9 +150,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -176,14 +168,10 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -201,13 +189,10 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); @@ -228,7 +213,6 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -250,7 +234,6 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -271,7 +254,6 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('vertexCount')); testClient.drain(done); }); }); From 7d5b6b08206515c7e3f7bcc9f2e12eb14eb2162e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 18:24:24 +0200 Subject: [PATCH 082/402] Lint changes and yarn.lock --- lib/cartodb/backends/stats.js | 2 +- lib/cartodb/controllers/map.js | 7 ++-- package.json | 1 + test/acceptance/multilayer.js | 2 +- test/acceptance/stats/multilayer_stats.js | 6 ++-- yarn.lock | 40 +++++++++-------------- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index d2e989e3..d8f83c72 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -2,7 +2,7 @@ var windshaftStats = require('windshaft-stats'); function StatsBackend(rendererCache) { this.rendererCache = rendererCache; -}; +} module.exports = StatsBackend; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f3211190..f52d839d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -256,7 +256,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, rendererParams, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, + mapConfigProvider.analysesResults, + rendererParams, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -279,7 +281,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); }; -MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { +MapController.prototype.afterLayergroupCreate = +function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { var self = this; var username = req.context.user; diff --git a/package.json b/package.json index 9bbb5125..b14b07d3 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "strftime": "~0.8.2" }, "scripts": { + "lint": "jshint lib test", "preinstall": "make pre-install", "test": "make test-all" }, diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 275ebab7..210418ac 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1041,7 +1041,7 @@ describe(suiteName, function() { ); }); } - + // See https://github.com/CartoDB/Windshaft-cartodb/issues/91 // and https://github.com/CartoDB/Windshaft-cartodb/issues/38 it("tiles for private tables can be fetched with api_key", function(done) { diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js index 4571bf6a..bfd76394 100644 --- a/test/acceptance/stats/multilayer_stats.js +++ b/test/acceptance/stats/multilayer_stats.js @@ -49,8 +49,10 @@ describe('multilayer stats disabled', function() { var cartocssVersion = '2.3.0'; var cartocss = '#layer { line-width:16; }'; - var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; - var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " + + "st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " + + "st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; var httpLayer = { type: 'http', diff --git a/yarn.lock b/yarn.lock index 1addd775..e8a320d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -223,6 +223,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -231,14 +239,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#0.15.1-cdb1": - version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -738,7 +738,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@3.2.3: +glob@3.2.3, "glob@~ 3.2.1": version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -777,13 +777,6 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" -"glob@~ 3.2.1": - version "3.2.11" - resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" - dependencies: - inherits "2" - minimatch "0.3" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1227,13 +1220,6 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -minimatch@0.3: - version "0.3.0" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" - dependencies: - lru-cache "2" - sigmund "~1.0.0" - minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -1523,7 +1509,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2275,6 +2261,12 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" +windshaft-stats@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/windshaft-stats/-/windshaft-stats-0.0.1.tgz#5af57a2ea71c67cdf6d8628941c88b826100d7cb" + dependencies: + debug "~2.2.0" + windshaft@cartodb/windshaft#548-vector-buffer-size: version "3.1.1" resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" From daf19c5e27479e5c8acfc5b6c73ef593cd9b1552 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 10 May 2017 17:17:01 +0200 Subject: [PATCH 083/402] Stats backend only provides stats not metadata --- lib/cartodb/backends/stats.js | 16 ++++++------- lib/cartodb/controllers/map.js | 24 +++++-------------- .../provider/dummy-mapconfig-provider.js | 20 ---------------- lib/cartodb/server.js | 2 +- .../stats/mapnik_stats_layergroup.js | 3 +++ test/acceptance/stats/multilayer_stats.js | 12 ++++++---- 6 files changed, 25 insertions(+), 52 deletions(-) delete mode 100644 lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index d8f83c72..374de6ff 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -1,20 +1,18 @@ var windshaftStats = require('windshaft-stats'); -function StatsBackend(rendererCache) { - this.rendererCache = rendererCache; +function StatsBackend() { } module.exports = StatsBackend; -StatsBackend.prototype.getStats = function(params, dbConnection, mapConfigProvider, callback) { +StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; - var layerStats; - if (layerMetadataEnabled) { - layerStats = windshaftStats(); - layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; + if (layerMetadataEnabled && layerStatsEnabled) { + var layerStats = windshaftStats(); + layerStats.getStats(mapConfig, params, dbConnection, callback); } else { - layerStats = windshaftStats('torque'); - layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + callback(null, null); } }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f52d839d..353b459d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -19,7 +19,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); -var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconfig-provider'); + /** * @param {AuthApi} authApi @@ -364,30 +364,18 @@ function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callb return null; }, - function fetchLayersMetadata(err) { + function fetchLayersStats(err) { assert.ifError(err); var next = this; - var mapConfigProvider = new DummyMapConfigProvider(mapconfig, rendererParams); - self.statsBackend.getStats(rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { + self.statsBackend.getStats(mapconfig, rendererParams, dbConnection, function(err, layersStats) { if (err) { return next(err); } if (layersStats) { - layergroup.metadata = layergroup.metadata || {}; - layergroup.metadata.layers = layersStats; - - // backwards compatibility for torque - var torqueMetadata = layersStats.reduce(function(acc, layer, layerId) { - if (layer.type === 'torque') { - acc[layerId] = layer.meta; - } - return acc; - }, {}); - if (Object.keys(torqueMetadata).length) { - layergroup.metadata.torque = torqueMetadata; - } + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); } - return next(); }); }, diff --git a/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js b/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js deleted file mode 100644 index 0b8942c5..00000000 --- a/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js +++ /dev/null @@ -1,20 +0,0 @@ -var util = require('util'); -var MapStoreMapConfigProvider = require('windshaft').model.provider.MapStoreMapConfig; - -function DummyMapConfigProvider(mapConfig, params) { - MapStoreMapConfigProvider.call(this, undefined, params); - - this.mapConfig = mapConfig; -} - -util.inherits(DummyMapConfigProvider, MapStoreMapConfigProvider); - -module.exports = DummyMapConfigProvider; - -DummyMapConfigProvider.prototype.setParams = function(params) { - this.params = params; -}; - -DummyMapConfigProvider.prototype.getMapConfig = function(callback) { - return callback(null, this.mapConfig, this.params, {}); -}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 5ce31f13..8cf42220 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -152,7 +152,7 @@ module.exports = function(serverOptions) { var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); - var statsBackend = new StatsBackend(rendererCache); + var statsBackend = new StatsBackend(); var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 4a327a01..1feb477e 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -6,11 +6,14 @@ var TestClient = require('../../support/test-client'); describe('Create mapnik layergroup', function() { before(function() { this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; global.environment.enabledFeatures.layerMetadata = true; + global.environment.enabledFeatures.layerStats = true; }); after(function() { global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); var cartocssVersion = '2.3.0'; diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js index bfd76394..ced63b0e 100644 --- a/test/acceptance/stats/multilayer_stats.js +++ b/test/acceptance/stats/multilayer_stats.js @@ -6,11 +6,15 @@ var TestClient = require('../../support/test-client'); describe('multilayer stats disabled', function() { before(function () { - global.environment.enabledFeatures.layerMetadata = false; + this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; + global.environment.enabledFeatures.layerMetadata = true; + global.environment.enabledFeatures.layerStats = false; }); after(function () { - global.environment.enabledFeatures.layerMetadata = true; + global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); @@ -28,9 +32,9 @@ describe('multilayer stats disabled', function() { assert.ifError(err); layergroup.metadata.layers.forEach(function (layer) { if (layer.type !== 'torque' && layer.type !== 'mapnik') { - assert.ok('stats' in layer.meta); + assert.ok(!('stats' in layer.meta)); } else if (layer.type !== 'torque') { - assert.ok('stats' in layer.meta); + assert.ok(!('stats' in layer.meta)); assert.ok('cartocss' in layer.meta); } else { assert.ok('cartocss' in layer.meta); From 968677e27566f386bffb9325a14b795cfc8b03ef Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 10 May 2017 17:26:12 +0200 Subject: [PATCH 084/402] package.json poiting to staging branches --- package.json | 4 ++-- yarn.lock | 42 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index b14b07d3..6d9065a7 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#548-vector-buffer-size", - "windshaft-stats": "0.0.1", + "windshaft": "cartodb/windshaft#644_layergroup_stats", + "windshaft-stats": "cartodb/windshaft-stats#adapt_existing_code", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index e8a320d2..cbdecfba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,6 +254,15 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" +cartodb-psql@~0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" + dependencies: + debug "~2.2.0" + pg cartodb/node-postgres#6.1.2-cdb1 + step "~0.0.6" + underscore "~1.6.0" + cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" @@ -402,6 +411,10 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" +d3-queue@~3.0.5: + version "3.0.7" + resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -430,6 +443,12 @@ debug@^1.0.4: dependencies: ms "0.6.2" +debug@~2.6.6: + version "2.6.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.6.tgz#a9fa6fbe9ca43cf1e79f73b75c0189cbb7d6db5a" + dependencies: + ms "0.7.3" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1293,6 +1312,10 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" +ms@0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1509,7 +1532,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2002,6 +2025,10 @@ step@~0.0.5, step@~0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" +step@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" + strftime@~0.8.2: version "0.8.4" resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" @@ -2261,15 +2288,18 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@0.0.1: +windshaft-stats@cartodb/windshaft-stats#adapt_existing_code: version "0.0.1" - resolved "https://registry.yarnpkg.com/windshaft-stats/-/windshaft-stats-0.0.1.tgz#5af57a2ea71c67cdf6d8628941c88b826100d7cb" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/f56056e8f212d133a68340b0367a6e4a7c721c7f" dependencies: - debug "~2.2.0" + cartodb-psql "~0.8.0" + d3-queue "~3.0.5" + debug "~2.6.6" + step "~1.0.0" -windshaft@cartodb/windshaft#548-vector-buffer-size: +windshaft@cartodb/windshaft#644_layergroup_stats: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/1f318899ba04f770d6bb14256b49a6d8fc0091b0" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 2c9d30e04279f59b0cdb7da03301743f0c6f23db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 10 May 2017 17:49:28 +0200 Subject: [PATCH 085/402] Be more flexible validating buffer-size customization --- .../adapter/mapconfig-named-map-adapter.js | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js index 76e8f3f5..3010db4a 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js @@ -1,35 +1,22 @@ -var _ = require('underscore'); - function MapConfigNamedMapAdapter() { } module.exports = MapConfigNamedMapAdapter; +var formats = ['png', 'png32', 'mvt', 'grid.json', 'geojson']; + MapConfigNamedMapAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { - if (context.templateParams && - context.templateParams.buffersize && - isValidBufferSize(context.templateParams.buffersize)) { - requestMapConfig.buffersize = context.templateParams.buffersize; + if (!context.templateParams || !context.templateParams.buffersize) { + return callback(null, requestMapConfig); } + formats.forEach(function (format) { + if (Number.isFinite(context.templateParams.buffersize[format])) { + requestMapConfig.buffersize[format] = context.templateParams.buffersize[format]; + } + }); + process.nextTick(function () { callback(null, requestMapConfig); }); }; - -function isValidBufferSize (bufferSize) { - var formats = ['png', 'png32', 'mvt', 'grid.json', 'geojson']; - - if (!_.isObject(bufferSize) || (_.isArray(bufferSize) || _.isFunction(bufferSize))) { - return false; - } - - for (var index = 0; index < formats.length; index++) { - var bufferSizeByFormat = bufferSize[formats[index]]; - if (bufferSizeByFormat && !Number.isFinite(bufferSizeByFormat)) { - return false; - } - } - - return true; -} From 53d1b2fbbffc7ebf0236f34dcb34109dda29243e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 10 May 2017 18:16:22 +0200 Subject: [PATCH 086/402] Rename mapconfig-named-map-adapter by mapconfig-buffer-size-adapter --- ...amed-map-adapter.js => mapconfig-buffer-size-adapter.js} | 6 +++--- lib/cartodb/server.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) rename lib/cartodb/models/mapconfig/adapter/{mapconfig-named-map-adapter.js => mapconfig-buffer-size-adapter.js} (71%) diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js similarity index 71% rename from lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js rename to lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js index 3010db4a..e362375b 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-map-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -1,11 +1,11 @@ -function MapConfigNamedMapAdapter() { +function MapConfigBufferSizeAdapter() { } -module.exports = MapConfigNamedMapAdapter; +module.exports = MapConfigBufferSizeAdapter; var formats = ['png', 'png32', 'mvt', 'grid.json', 'geojson']; -MapConfigNamedMapAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { +MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { if (!context.templateParams || !context.templateParams.buffersize) { return callback(null, requestMapConfig); } diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 22b14f5b..273f34c1 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -35,7 +35,7 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter'); -var MapConfigNamedMapAdapter = require('./models/mapconfig/adapter/mapconfig-named-map-adapter'); +var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter'); var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter'); var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); @@ -156,7 +156,7 @@ module.exports = function(serverOptions) { var mapConfigAdapter = new MapConfigAdapter( new MapConfigNamedLayersAdapter(templateMaps, pgConnection), - new MapConfigNamedMapAdapter(), + new MapConfigBufferSizeAdapter(), new SqlWrapMapConfigAdapter(), new DataviewsWidgetsAdapter(), new AnalysisMapConfigAdapter(analysisBackend), From 5a44d6c5470eee86aa23368a663a9b19d4a495d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 10 May 2017 18:35:30 +0200 Subject: [PATCH 087/402] Drop geojson support for buffersize customization --- .../adapter/mapconfig-buffer-size-adapter.js | 2 +- test/acceptance/buffer-size-format.js | 98 ------------------- 2 files changed, 1 insertion(+), 99 deletions(-) diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js index e362375b..02ce6443 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -3,7 +3,7 @@ function MapConfigBufferSizeAdapter() { module.exports = MapConfigBufferSizeAdapter; -var formats = ['png', 'png32', 'mvt', 'grid.json', 'geojson']; +var formats = ['png', 'png32', 'mvt', 'grid.json']; MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { if (!context.templateParams || !context.templateParams.buffersize) { diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index f573057c..e62ee4a3 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -100,30 +100,6 @@ describe('buffer size per format', function () { callback(); } }, - { - desc: 'should get geojson tile using buffer-size 0 overriden by template params', - coords: { z: 7, x: 64, y: 48 }, - format: 'geojson', - fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson', - mapConfig: createMapConfig({ geojson: 0 }), - assert: function (tile, callback) { - var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); - assert.equal(tile.features.length, dataFixture.features.length); - callback(); - } - }, - { - desc: 'should get geojson tile using buffer-size 128 overriden by template params', - coords: { z: 7, x: 64, y: 48 }, - format: 'geojson', - fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson', - mapConfig: createMapConfig({ geojson: 128 }), - assert: function (tile, callback) { - var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); - assert.equal(tile.features.length, dataFixture.features.length); - callback(); - } - }, { desc: 'should get grid.json tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, @@ -238,46 +214,6 @@ describe('buffer size per format for named maps', function () { assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); } }, - { - desc: 'should get geojson tile using buffer-size 0 overriden by template params', - coords: { z: 7, x: 64, y: 48 }, - format: 'geojson', - placeholders: { buffersize_geojson: 0 }, - fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson', - template: createBufferSizeTemplate('named-default-buffer-size-by-format-geojson', { - geojson: '<%= buffersize_geojson %>' - }, { - "buffersize_geojson": { - "type": "number", - "default": "0" - } - }), - assert: function (tile, callback) { - var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); - assert.equal(tile.features.length, dataFixture.features.length); - callback(); - } - }, - { - desc: 'should get geojson tile using buffer-size 128 overriden by template params', - coords: { z: 7, x: 64, y: 48 }, - format: 'geojson', - placeholders: { buffersize_geojson: 128 }, - fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson', - template: createBufferSizeTemplate('named-custom-buffer-size-by-format-geojson', { - geojson: '<%= buffersize_geojson %>' - }, { - "buffersize_geojson": { - "type": "number", - "default": "0" - } - }), - assert: function (tile, callback) { - var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); - assert.equal(tile.features.length, dataFixture.features.length); - callback(); - } - }, { desc: 'should get grid.json tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, @@ -420,40 +356,6 @@ describe('buffer size per format for named maps w/o placeholders', function () { callback(); } }, - { - desc: 'should get geojson tile using buffer-size 0 overriden by template params', - coords: { z: 7, x: 64, y: 48 }, - format: 'geojson', - placeholders: { - buffersize: { - geojson: 0 - } - }, - fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson', - template: createBufferSizeTemplate('named-no-buffer-size-geojson-0', {}, {}), - assert: function (tile, callback) { - var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); - assert.equal(tile.features.length, dataFixture.features.length); - callback(); - } - }, - { - desc: 'should get geojson tile using buffer-size 128 overriden by template params', - coords: { z: 7, x: 64, y: 48 }, - format: 'geojson', - placeholders: { - buffersize: { - geojson: 128 - } - }, - fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson', - template: createBufferSizeTemplate('named-no-buffer-size-geojson-128', {}, {}), - assert: function (tile, callback) { - var dataFixture = JSON.parse(fs.readFileSync(this.fixturePath)); - assert.equal(tile.features.length, dataFixture.features.length); - callback(); - } - }, { desc: 'should get grid.json tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, From 61f9ea6e865c45e335608a0ea757c92465f4af56 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 12:47:35 +0200 Subject: [PATCH 088/402] Changed stats name from featureCount to estimatedFeatureCount --- .../stats/mapnik_stats_layergroup.js | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 1feb477e..8bee7fee 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -152,8 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -170,11 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -191,12 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -215,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -236,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -256,7 +251,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); + assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); testClient.drain(done); }); }); From 66a898cdc201283918384804b6d6269334a0ce5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 11 May 2017 12:55:53 +0200 Subject: [PATCH 089/402] Upgrade camshaft to get error node-id --- package.json | 2 +- test/acceptance/analysis/error-cases.js | 65 +++++++++++++++++++++++++ yarn.lock | 6 +-- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index e7a6993e..7d09c6e0 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.54.3", + "camshaft": "cartodb/camshaft#master", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/test/acceptance/analysis/error-cases.js b/test/acceptance/analysis/error-cases.js index 3bde8442..b79e1046 100644 --- a/test/acceptance/analysis/error-cases.js +++ b/test/acceptance/analysis/error-cases.js @@ -373,5 +373,70 @@ describe('analysis-layers error cases', function() { }); }); + it('should return "function does not exist" indicating the node_id and context', function(done) { + var mapConfig = createMapConfig([{ + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": '#polygons { polygon-fill: red; }', + "cartocss_version": "2.3.0" + } + }], {}, [{ + "id": "HEAD", + "type": "buffer", + "params": { + "source": { + "id": "HEAD2", + "type": "buffer", + "params": { + "source": { + "id": "HEAD3", + "type": 'deprecated-sql-function', + "params": { + "id": "HEAD4", + "function_name": 'DEP_EXT_does_not_exist_fn', + "primary_source": { + "type": 'source', + "params": { + "query": "select * from populated_places_simple_reduced" + } + }, + "function_args": ['wadus'] + } + }, + "radius": 10 + } + }, + "radius": 10 + } + }]); + + var testClient = new TestClient(mapConfig, 1234); + + testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + assert.ok(!err, err); + + assert.equal(layergroupResult.errors.length, 1); + assert.equal( + layergroupResult.errors[0], + 'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist' + ); + + assert.equal(layergroupResult.errors_with_context[0].type, 'analysis'); + assert.equal( + layergroupResult.errors_with_context[0].message, + 'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist' + ); + assert.equal(layergroupResult.errors_with_context[0].analysis.id, 'HEAD'); + assert.equal(layergroupResult.errors_with_context[0].analysis.type, 'buffer'); + assert.equal(layergroupResult.errors_with_context[0].analysis.node_id, 'HEAD3'); + + testClient.drain(done); + }); + }); + + }); diff --git a/yarn.lock b/yarn.lock index fb7b4d06..a4379056 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.54.3: - version "0.54.3" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.3.tgz#ba0b0fc00c5f82dc5d6b7d7649ed3412f6a9902b" +camshaft@cartodb/camshaft#master: + version "0.54.4" + resolved "https://codeload.github.com/cartodb/camshaft/tar.gz/76bcf3dfe41f5aa6e2b181de22628fb30e1a880e" dependencies: async "^1.5.2" bunyan "1.8.1" From b739db1023e95fb04a0adf5dca417c956caf9dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 11 May 2017 13:19:14 +0200 Subject: [PATCH 090/402] Use released version of camshaft --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 7d09c6e0..204d5491 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "cartodb/camshaft#master", + "camshaft": "0.54.4", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index a4379056..de05fdf3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@cartodb/camshaft#master: +camshaft@0.54.4: version "0.54.4" - resolved "https://codeload.github.com/cartodb/camshaft/tar.gz/76bcf3dfe41f5aa6e2b181de22628fb30e1a880e" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.4.tgz#3e1e96cda36661ebc01b64dadcc6387b02d69739" dependencies: async "^1.5.2" bunyan "1.8.1" From 787ca1607a26a50eeb540e137167f608ce6dcf5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 11 May 2017 13:37:43 +0200 Subject: [PATCH 091/402] Release 3.6.6 --- NEWS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index a4ef9bdd..2b8e134f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,10 @@ # Changelog ## 3.6.6 -Released 2017-mm-dd +Released 2017-05-11 + +Announcements: + - Upgrades camshaft to [0.54.4](https://github.com/CartoDB/camshaft/releases/tag/0.54.4). ## 3.6.5 From f9c8178d9999712982ba71b05c161bfc5d7997f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 11 May 2017 13:46:34 +0200 Subject: [PATCH 092/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2b8e134f..ca420d85 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.6.7 +Released 2017-mm-dd + + ## 3.6.6 Released 2017-05-11 diff --git a/package.json b/package.json index 204d5491..00e2f520 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.6", + "version": "3.6.7", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 35df0c3a686ed608295eafaf7a455bc500222406 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 14:26:00 +0200 Subject: [PATCH 093/402] Change from branch to commits for staging dependencies --- package.json | 4 ++-- yarn.lock | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 6d9065a7..217812fc 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#644_layergroup_stats", - "windshaft-stats": "cartodb/windshaft-stats#adapt_existing_code", + "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", + "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index cbdecfba..57e245f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1532,7 +1532,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2288,18 +2288,18 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#adapt_existing_code: +windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/f56056e8f212d133a68340b0367a6e4a7c721c7f" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" dependencies: cartodb-psql "~0.8.0" d3-queue "~3.0.5" debug "~2.6.6" step "~1.0.0" -windshaft@cartodb/windshaft#644_layergroup_stats: +windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/1f318899ba04f770d6bb14256b49a6d8fc0091b0" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From f6fff6953e2100414f7f7b9dad8352626da0ddb0 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:50:57 +0200 Subject: [PATCH 094/402] We leave only one feature flag for stats --- lib/cartodb/backends/stats.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 374de6ff..489c1740 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -7,9 +7,8 @@ module.exports = StatsBackend; StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; - var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; - if (layerMetadataEnabled && layerStatsEnabled) { + if (layerStatsEnabled) { var layerStats = windshaftStats(); layerStats.getStats(mapConfig, params, dbConnection, callback); } else { From 44cca385381f39ae1752440471ea1f5355e97d9c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:51:45 +0200 Subject: [PATCH 095/402] Change to tests for layer stats because now uses CDB_EstimateRowCount fuction --- .../stats/mapnik_stats_layergroup.js | 30 +++++++++---------- test/support/prepare_db.sh | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 8bee7fee..77288d51 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -22,7 +22,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer1 = { type: 'mapnik', options: { - sql: 'select * from test_table limit 2', + sql: 'select * from test_table limit 1', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -40,7 +40,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer3 = { type: 'mapnik', options: { - sql: 'select * from test_table_3 limit 2', + sql: 'select * from test_table_3 limit 3', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3); testClient.drain(done); }); }); @@ -152,7 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -169,9 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -188,10 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -210,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -231,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 4fbf4b7a..dcf1cba7 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' - REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount' CURL_ARGS="" for i in ${REMOTE_SQL_SCRIPTS} From 7596df96ed836ac3a0029c7959eb09392ce8f9e5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:58:13 +0200 Subject: [PATCH 096/402] package.json for staging --- package.json | 4 ++-- yarn.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 217812fc..adaa4cc1 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", - "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", + "windshaft": "cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd", + "windshaft-stats": "cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 57e245f1..0f4c6c2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2288,18 +2288,18 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: +windshaft-stats@cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d: version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/5f41e0a69763692f45f76cbae7f1e784186e1e7d" dependencies: cartodb-psql "~0.8.0" d3-queue "~3.0.5" debug "~2.6.6" step "~1.0.0" -windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: +windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/0675de5edcb2a24179dcd6224c3f309b68f17bbd" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 1a7fd9bf3170aba93d8a3e02715789df2679abe7 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 12:47:35 +0200 Subject: [PATCH 097/402] Changed stats name from featureCount to estimatedFeatureCount --- .../stats/mapnik_stats_layergroup.js | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 1feb477e..8bee7fee 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -152,8 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -170,11 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -191,12 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -215,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -236,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -256,7 +251,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); + assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); testClient.drain(done); }); }); From 522c86e6f2c3f59f09da0c7e56013d0f660597d6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 14:26:00 +0200 Subject: [PATCH 098/402] Change from branch to commits for staging dependencies --- package.json | 4 ++-- yarn.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index b14b07d3..217812fc 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#548-vector-buffer-size", - "windshaft-stats": "0.0.1", + "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", + "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index e8a320d2..8d02a27e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2261,15 +2261,15 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@0.0.1: +windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: version "0.0.1" - resolved "https://registry.yarnpkg.com/windshaft-stats/-/windshaft-stats-0.0.1.tgz#5af57a2ea71c67cdf6d8628941c88b826100d7cb" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" dependencies: debug "~2.2.0" -windshaft@cartodb/windshaft#548-vector-buffer-size: +windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 59585b5cd9fca14d21cdbdae96557c4272f00e2b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:50:57 +0200 Subject: [PATCH 099/402] We leave only one feature flag for stats --- lib/cartodb/backends/stats.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 374de6ff..489c1740 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -7,9 +7,8 @@ module.exports = StatsBackend; StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; - var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; - if (layerMetadataEnabled && layerStatsEnabled) { + if (layerStatsEnabled) { var layerStats = windshaftStats(); layerStats.getStats(mapConfig, params, dbConnection, callback); } else { From 5417933eccdfac42e6d9fc4770c3268da966439b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:51:45 +0200 Subject: [PATCH 100/402] Change to tests for layer stats because now uses CDB_EstimateRowCount fuction --- .../stats/mapnik_stats_layergroup.js | 30 +++++++++---------- test/support/prepare_db.sh | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 8bee7fee..77288d51 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -22,7 +22,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer1 = { type: 'mapnik', options: { - sql: 'select * from test_table limit 2', + sql: 'select * from test_table limit 1', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -40,7 +40,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer3 = { type: 'mapnik', options: { - sql: 'select * from test_table_3 limit 2', + sql: 'select * from test_table_3 limit 3', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3); testClient.drain(done); }); }); @@ -152,7 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -169,9 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -188,10 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -210,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -231,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 4fbf4b7a..dcf1cba7 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' - REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount' CURL_ARGS="" for i in ${REMOTE_SQL_SCRIPTS} From 5969c99e8a5168f2afb0c6f2126c381562ca70a2 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 16:39:54 +0200 Subject: [PATCH 101/402] Removed not used parameters for layer stats --- lib/cartodb/backends/stats.js | 6 +++--- lib/cartodb/controllers/map.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 489c1740..eb800e3c 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -5,13 +5,13 @@ function StatsBackend() { module.exports = StatsBackend; -StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { +StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; if (layerStatsEnabled) { var layerStats = windshaftStats(); - layerStats.getStats(mapConfig, params, dbConnection, callback); + layerStats.getStats(mapConfig, dbConnection, callback); } else { - callback(null, null); + return callback(null, null); } }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 353b459d..cd5bbed3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -367,7 +367,7 @@ function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callb function fetchLayersStats(err) { assert.ifError(err); var next = this; - self.statsBackend.getStats(mapconfig, rendererParams, dbConnection, function(err, layersStats) { + self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) { if (err) { return next(err); } From 422867762b236ef1eccb80f5da5694d4d625ba9f Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 17:14:07 +0200 Subject: [PATCH 102/402] package.json for staging --- package.json | 4 ++-- yarn.lock | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 217812fc..de4f6259 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", - "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", + "windshaft": "cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd", + "windshaft-stats": "cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 8d02a27e..3092b78f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,6 +254,15 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" +cartodb-psql@~0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" + dependencies: + debug "~2.2.0" + pg cartodb/node-postgres#6.1.2-cdb1 + step "~0.0.6" + underscore "~1.6.0" + cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" @@ -402,6 +411,10 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" +d3-queue@~3.0.5: + version "3.0.7" + resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -430,6 +443,12 @@ debug@^1.0.4: dependencies: ms "0.6.2" +debug@~2.6.6: + version "2.6.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.6.tgz#a9fa6fbe9ca43cf1e79f73b75c0189cbb7d6db5a" + dependencies: + ms "0.7.3" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1293,6 +1312,10 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" +ms@0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1509,7 +1532,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2261,15 +2284,17 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: +windshaft-stats@cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f: version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/16a973ba9addf88312ba92047e41108be2c3434f" dependencies: - debug "~2.2.0" + cartodb-psql "~0.8.0" + d3-queue "~3.0.5" + debug "~2.6.6" -windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: +windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/0675de5edcb2a24179dcd6224c3f309b68f17bbd" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 29361f5392a1f09b4975cd4db9b51c113cd4afa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 16 May 2017 12:07:17 +0200 Subject: [PATCH 103/402] Use windhshaft with latest changes for layer stats --- package.json | 2 +- yarn.lock | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index adaa4cc1..8ad0eda7 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd", + "windshaft": "cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea", "windshaft-stats": "cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d", "yargs": "~5.0.0" }, diff --git a/yarn.lock b/yarn.lock index 0f4c6c2a..05642fd5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,7 +254,7 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" -cartodb-psql@~0.8.0: +cartodb-psql@0.8.0, cartodb-psql@~0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" dependencies: @@ -2117,17 +2117,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#buffer-size: +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: version "2.3.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/9df4bde07f3cf9ffbda42657066559e2b8875e1b" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#fix-buffer-size-0: +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb1: version "0.6.18-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/c67400573572e836609dd75a7acec8f414a50dbe" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2297,14 +2297,14 @@ windshaft-stats@cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d debug "~2.6.6" step "~1.0.0" -windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: - version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/0675de5edcb2a24179dcd6224c3f309b68f17bbd" +windshaft@cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea: + version "3.1.3" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/515efb2d9237a1511b81a61fd6f7524184df38ea" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.7.1" + cartodb-psql "0.8.0" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" @@ -2316,8 +2316,8 @@ windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#buffer-size - tilelive-mapnik cartodb/tilelive-mapnik#fix-buffer-size-0 + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb1 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb1 torque.js "~2.11.0" underscore "~1.6.0" From e13ae8d5aff6faea67c2594021f56ea9c8f8e29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 11:40:18 +0200 Subject: [PATCH 104/402] Do not make optional layer param in URL template --- lib/cartodb/controllers/layergroup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index fb1aab9b..79268d6e 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -56,7 +56,7 @@ LayergroupController.prototype.register = function(app) { this.tile.bind(this)); app.get(app.base_url_mapconfig + - '/:token/:z/:x/:y.(:format)', cors(), userMiddleware, + '/:token/:z/:x/:y.:format', cors(), userMiddleware, this.tile.bind(this)); app.get(app.base_url_mapconfig + From 13b1978d49ad5a68c1d683bf65796f1f4aa7d564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 11:40:53 +0200 Subject: [PATCH 105/402] Include layer param to reach the right tile for grid.json --- test/acceptance/buffer-size-format.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index e62ee4a3..d5157585 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -104,6 +104,7 @@ describe('buffer size per format', function () { desc: 'should get grid.json tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', + layers: [0], fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', mapConfig: createMapConfig({ 'grid.json': 0 }), assert: function (tile, callback) { @@ -114,6 +115,7 @@ describe('buffer size per format', function () { desc: 'should get grid.json tile using buffer-size 128 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', + layers: [0], fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', mapConfig: createMapConfig({ 'grid.json': 128 }), assert: function (tile, callback) { @@ -126,7 +128,11 @@ describe('buffer size per format', function () { it(test.desc, function (done) { var testClient = new TestClient(test.mapConfig, 1234); var coords = test.coords; - testClient.getTile(coords.z, coords.x, coords.y, { format: test.format }, function (err, res, tile) { + var options = { + format: test.format, + layers: test.layers + }; + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: // tile.save(test.fixturePath); @@ -218,6 +224,7 @@ describe('buffer size per format for named maps', function () { desc: 'should get grid.json tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', + layers: [0], placeholders: { buffersize_gridjson: 0 }, fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', { @@ -236,6 +243,7 @@ describe('buffer size per format for named maps', function () { desc: 'should get grid.json tile using buffer-size 128 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', + layers: [0], placeholders: { buffersize_gridjson: 128 }, fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', { @@ -258,7 +266,8 @@ describe('buffer size per format for named maps', function () { var coords = test.coords; var options = { format: test.format, - placeholders: test.placeholders + placeholders: test.placeholders, + layers: test.layers }; testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); @@ -360,6 +369,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get grid.json tile using buffer-size 0 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', + layers: [0], placeholders: { buffersize: { 'grid.json': 0 @@ -375,6 +385,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { desc: 'should get grid.json tile using buffer-size 128 overriden by template params', coords: { z: 7, x: 64, y: 48 }, format: 'grid.json', + layers: [0], placeholders: { buffersize: { 'grid.json': 128 @@ -394,7 +405,8 @@ describe('buffer size per format for named maps w/o placeholders', function () { var coords = test.coords; var options = { format: test.format, - placeholders: test.placeholders + placeholders: test.placeholders, + layers: test.layers }; testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); From c59996303d056e20cf0924dca1a0654c3cef997c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 12:04:11 +0200 Subject: [PATCH 106/402] Send stats for mvt tiles --- lib/cartodb/controllers/layergroup.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 79268d6e..560f4267 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -253,7 +253,8 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t grid_json: true, json_torque: true, torque_json: true, - png: true + png: true, + mvt: true }; var formatStat = 'invalid'; From 4602fb3ecfb420f4f7ac5089fbe3be5c200536fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 12:16:16 +0200 Subject: [PATCH 107/402] Send stats for png32 tiles --- lib/cartodb/controllers/layergroup.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 560f4267..e0c7a7a3 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -254,6 +254,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t json_torque: true, torque_json: true, png: true, + png32: true, mvt: true }; From 808c729a0e831d10ff8b85cda09ba38f17615e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 12:33:41 +0200 Subject: [PATCH 108/402] Now supported formats for buffer-size customization are bound to the adapter --- .../mapconfig/adapter/mapconfig-buffer-size-adapter.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js index 02ce6443..5b961ac4 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -1,16 +1,15 @@ function MapConfigBufferSizeAdapter() { + this.formats = ['png', 'png32', 'mvt', 'grid.json']; } module.exports = MapConfigBufferSizeAdapter; -var formats = ['png', 'png32', 'mvt', 'grid.json']; - MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { if (!context.templateParams || !context.templateParams.buffersize) { return callback(null, requestMapConfig); } - formats.forEach(function (format) { + this.formats.forEach(function (format) { if (Number.isFinite(context.templateParams.buffersize[format])) { requestMapConfig.buffersize[format] = context.templateParams.buffersize[format]; } From fb4ee61b8345519c17c9963222ecc60095604487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 12:55:05 +0200 Subject: [PATCH 109/402] Use setInmmediate vs process.nextTick --- .../models/mapconfig/adapter/mapconfig-buffer-size-adapter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js index 5b961ac4..3296d0ea 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -15,7 +15,7 @@ MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapCo } }); - process.nextTick(function () { + setImmediate(function () { callback(null, requestMapConfig); }); }; From c65518cf41c135cd0289b8fb87176915862d7004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 20:16:43 +0200 Subject: [PATCH 110/402] Get back layer-stats from windshaft-stats --- .../backends/layer-stats/empty-layer-stats.js | 16 ++ lib/cartodb/backends/layer-stats/factory.js | 23 +++ .../backends/layer-stats/layer-stats.js | 45 ++++++ .../layer-stats/mapnik-layer-stats.js | 28 ++++ .../backends/layer-stats/query-utils.js | 26 +++ .../layer-stats/torque-layer-stats.js | 16 ++ lib/cartodb/backends/stats.js | 5 +- package.json | 1 - .../layer-stats/mapnik-layer-stats.js | 153 ++++++++++++++++++ .../layer-stats/torque-layer-stats.js | 36 +++++ yarn.lock | 30 +--- 11 files changed, 347 insertions(+), 32 deletions(-) create mode 100644 lib/cartodb/backends/layer-stats/empty-layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/factory.js create mode 100644 lib/cartodb/backends/layer-stats/layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/mapnik-layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/query-utils.js create mode 100644 lib/cartodb/backends/layer-stats/torque-layer-stats.js create mode 100644 test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js create mode 100644 test/unit/cartodb/backends/layer-stats/torque-layer-stats.js diff --git a/lib/cartodb/backends/layer-stats/empty-layer-stats.js b/lib/cartodb/backends/layer-stats/empty-layer-stats.js new file mode 100644 index 00000000..0923b59c --- /dev/null +++ b/lib/cartodb/backends/layer-stats/empty-layer-stats.js @@ -0,0 +1,16 @@ +function EmptyLayerStats(types) { + this._types = types || {}; +} + +EmptyLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +EmptyLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + process.nextTick(function() { + callback(null, {}); + }); +}; + +module.exports = EmptyLayerStats; diff --git a/lib/cartodb/backends/layer-stats/factory.js b/lib/cartodb/backends/layer-stats/factory.js new file mode 100644 index 00000000..8aacdb5a --- /dev/null +++ b/lib/cartodb/backends/layer-stats/factory.js @@ -0,0 +1,23 @@ +var LayerStats = require('./layer-stats'); +var EmptyLayerStats = require('./empty-layer-stats'); +var MapnikLayerStats = require('./mapnik-layer-stats'); +var TorqueLayerStats = require('./torque-layer-stats'); + +module.exports = function LayerStatsFactory(type) { + var layerStatsIterator = []; + var selectedType = type || 'ALL'; + + if (selectedType === 'ALL') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true })); + layerStatsIterator.push(new MapnikLayerStats()); + layerStatsIterator.push(new TorqueLayerStats()); + } else if (selectedType === 'mapnik') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true })); + layerStatsIterator.push(new MapnikLayerStats()); + } else if (selectedType === 'torque') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true })); + layerStatsIterator.push(new TorqueLayerStats()); + } + + return new LayerStats(layerStatsIterator); +}; diff --git a/lib/cartodb/backends/layer-stats/layer-stats.js b/lib/cartodb/backends/layer-stats/layer-stats.js new file mode 100644 index 00000000..bc1787c2 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/layer-stats.js @@ -0,0 +1,45 @@ +var d3 = require('d3-queue'); + +function LayerStats(layerStatsIterator) { + this.layerStatsIterator = layerStatsIterator; +} + +LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) { + var self = this; + var stats = []; + + if (!mapConfig.getLayers().length) { + return callback(null, stats); + } + var metaQueue = d3.queue(mapConfig.getLayers().length); + mapConfig.getLayers().forEach(function (layer, layerId) { + var layerType = mapConfig.layerType(layerId); + + for (var i = 0; i < self.layerStatsIterator.length; i++) { + if (self.layerStatsIterator[i].is(layerType)) { + var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]); + metaQueue.defer(getStats, layer, dbConnection); + break; + } + } + }); + + metaQueue.awaitAll(function (err, results) { + if (err) { + return callback(err); + } + + if (!results) { + return callback(null, null); + } + + mapConfig.getLayers().forEach(function (layer, layerIndex) { + stats[layerIndex] = results[layerIndex]; + }); + + return callback(err, stats); + }); + +}; + +module.exports = LayerStats; diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..7dfd97af --- /dev/null +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,28 @@ +var queryUtils = require('./query-utils'); + +function MapnikLayerStats () { + this._types = { + mapnik: true, + cartodb: true + }; +} + +MapnikLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +MapnikLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); + // This query would gather stats for postgresql table if not exists + dbConnection.query(queryRowCountSql, function (err, res) { + if (err) { + return callback(null, {estimatedFeatureCount: -1}); + } else { + // We decided that the relation is 1 row == 1 feature + return callback(null, {estimatedFeatureCount: res.rows[0].rows}); + } + }); +}; + +module.exports = MapnikLayerStats; diff --git a/lib/cartodb/backends/layer-stats/query-utils.js b/lib/cartodb/backends/layer-stats/query-utils.js new file mode 100644 index 00000000..47d730f4 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/query-utils.js @@ -0,0 +1,26 @@ +function prepareQuery(sql) { + var affectedTableRegexCache = { + bbox: /!bbox!/g, + scale_denominator: /!scale_denominator!/g, + pixel_width: /!pixel_width!/g, + pixel_height: /!pixel_height!/g + }; + + return sql + .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') + .replace(affectedTableRegexCache.scale_denominator, '0') + .replace(affectedTableRegexCache.pixel_width, '1') + .replace(affectedTableRegexCache.pixel_height, '1'); +} + +module.exports.extractTableNames = function extractTableNames(query) { + return [ + 'SELECT * FROM CDB_QueryTablesText($windshaft$', + prepareQuery(query), + '$windshaft$) as tablenames' + ].join(''); +}; + +module.exports.getQueryRowCount = function getQueryRowEstimation(query) { + return 'select CDB_EstimateRowCount(\'' + query + '\') as rows'; +}; diff --git a/lib/cartodb/backends/layer-stats/torque-layer-stats.js b/lib/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..00b4def2 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,16 @@ +function TorqueLayerStats() { + this._types = { + torque: true + }; +} + +TorqueLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +TorqueLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + return callback(null, {}); +}; + +module.exports = TorqueLayerStats; diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index eb800e3c..20f89a0f 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -1,4 +1,4 @@ -var windshaftStats = require('windshaft-stats'); +var layerStats = require('./layer-stats/factory'); function StatsBackend() { } @@ -9,8 +9,7 @@ StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; if (layerStatsEnabled) { - var layerStats = windshaftStats(); - layerStats.getStats(mapConfig, dbConnection, callback); + layerStats().getStats(mapConfig, dbConnection, callback); } else { return callback(null, null); } diff --git a/package.json b/package.json index 4affa452..c49f1fe4 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "turbo-carto": "0.19.0", "underscore": "~1.6.0", "windshaft": "cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea", - "windshaft-stats": "cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..6e47297d --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,153 @@ +var assert = require('assert'); +var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +function getDbConnectionMock () { + return { + query: function(sql, callback) { + return callback(null, { + rows: [{rows: 1}] + }); + } + }; +} + +describe('mapnik-layer-stats', function() { + + beforeEach(function () { + this.dbConnectionMock = getDbConnectionMock(); + this.rendererCacheMock = {}; + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + } + ] + }; + + var testMapConfigTwoLayers = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + ] + }; + + var testMapConfigOneLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + ] + }; + + var testMapConfigTwoLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_3', 'test_table_4'] + } + }, + ] + }; + + it('should return 1 feature for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayers); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); + + it('should return 1 feature for one layers with two tables', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers and two tables', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); +}); diff --git a/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..c9adfad9 --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,36 @@ +var assert = require('assert'); +var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +describe('torque-layer-stats', function () { + + beforeEach(function () { + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'torque', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + } + }, + ] + }; + + it('should return torque stats for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layerId = 0; + var layer = mapConfig.getLayer(layerId); + var testSubject = new TorqueLayerStats(); + testSubject.getStats(layer, {}, function (err, result) { + assert.ifError(err); + assert.deepEqual({}, result); + done(); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 8d8ec0d0..a1b13067 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,7 +254,7 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" -cartodb-psql@0.8.0, cartodb-psql@~0.8.0: +cartodb-psql@0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" dependencies: @@ -411,10 +411,6 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" -d3-queue@~3.0.5: - version "3.0.7" - resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" - dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -443,12 +439,6 @@ debug@^1.0.4: dependencies: ms "0.6.2" -debug@~2.6.6: - version "2.6.6" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.6.tgz#a9fa6fbe9ca43cf1e79f73b75c0189cbb7d6db5a" - dependencies: - ms "0.7.3" - decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1312,10 +1302,6 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" -ms@0.7.3: - version "0.7.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" - mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1532,7 +1518,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2025,10 +2011,6 @@ step@~0.0.5, step@~0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" -step@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" - strftime@~0.8.2: version "0.8.4" resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" @@ -2288,14 +2270,6 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f: - version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/16a973ba9addf88312ba92047e41108be2c3434f" - dependencies: - cartodb-psql "~0.8.0" - d3-queue "~3.0.5" - debug "~2.6.6" - windshaft@cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea: version "3.1.3" resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/515efb2d9237a1511b81a61fd6f7524184df38ea" From ed56094be2c3fc214dd7a3ed8f143df8cf1944b9 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 18 May 2017 11:51:12 +0200 Subject: [PATCH 111/402] PR changes --- lib/cartodb/backends/layer-stats/empty-layer-stats.js | 2 +- lib/cartodb/backends/layer-stats/layer-stats.js | 4 ++-- .../backends/layer-stats/mapnik-layer-stats.js | 2 +- lib/cartodb/controllers/map.js | 11 ++++------- .../{backends/layer-stats => utils}/query-utils.js | 0 5 files changed, 8 insertions(+), 11 deletions(-) rename lib/cartodb/{backends/layer-stats => utils}/query-utils.js (100%) diff --git a/lib/cartodb/backends/layer-stats/empty-layer-stats.js b/lib/cartodb/backends/layer-stats/empty-layer-stats.js index 0923b59c..0760c0b6 100644 --- a/lib/cartodb/backends/layer-stats/empty-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/empty-layer-stats.js @@ -8,7 +8,7 @@ EmptyLayerStats.prototype.is = function (type) { EmptyLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - process.nextTick(function() { + setImmediate(function() { callback(null, {}); }); }; diff --git a/lib/cartodb/backends/layer-stats/layer-stats.js b/lib/cartodb/backends/layer-stats/layer-stats.js index bc1787c2..2464fb22 100644 --- a/lib/cartodb/backends/layer-stats/layer-stats.js +++ b/lib/cartodb/backends/layer-stats/layer-stats.js @@ -1,4 +1,4 @@ -var d3 = require('d3-queue'); +var queue = require('queue-async'); function LayerStats(layerStatsIterator) { this.layerStatsIterator = layerStatsIterator; @@ -11,7 +11,7 @@ LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) { if (!mapConfig.getLayers().length) { return callback(null, stats); } - var metaQueue = d3.queue(mapConfig.getLayers().length); + var metaQueue = queue(mapConfig.getLayers().length); mapConfig.getLayers().forEach(function (layer, layerId) { var layerType = mapConfig.layerType(layerId); diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 7dfd97af..c060f964 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,4 +1,4 @@ -var queryUtils = require('./query-utils'); +var queryUtils = require('../../utils/query-utils'); function MapnikLayerStats () { this._types = { diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index cd5bbed3..3bb72574 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -166,7 +166,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, req.params, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this); }, function finish(err, layergroup) { if (err) { @@ -219,8 +219,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn var mapConfigProvider; var mapConfig; - var rendererParams; - step( function setupParams(){ self.req2params(req, this); @@ -244,9 +242,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); mapConfigProvider.getMapConfig(this); }, - function createLayergroup(err, mapConfig_, _rendererParams) { + function createLayergroup(err, mapConfig_, rendererParams) { assert.ifError(err); - rendererParams = _rendererParams; mapConfig = mapConfig_; self.mapBackend.createLayergroup( mapConfig, rendererParams, @@ -258,7 +255,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn assert.ifError(err); self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, - rendererParams, this); + this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -282,7 +279,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }; MapController.prototype.afterLayergroupCreate = -function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { +function(req, res, mapconfig, layergroup, analysesResults, callback) { var self = this; var username = req.context.user; diff --git a/lib/cartodb/backends/layer-stats/query-utils.js b/lib/cartodb/utils/query-utils.js similarity index 100% rename from lib/cartodb/backends/layer-stats/query-utils.js rename to lib/cartodb/utils/query-utils.js From fd178bcf71d4ee5b7eee9d1d0f39483192e63ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 13:14:51 +0200 Subject: [PATCH 112/402] Upgrade windshaft to 3.2.0 --- package.json | 2 +- yarn.lock | 62 ++++++++++++++++++++-------------------------------- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/package.json b/package.json index c2a0721f..d33bb526 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#548-vector-buffer-size", + "windshaft": "3.2.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 8a9fd937..a203d58e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.54.4: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,7 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@cartodb/carto#0.15.1-cdb3: +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -231,14 +239,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#0.15.1-cdb1": - version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -738,7 +738,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@3.2.3: +glob@3.2.3, "glob@~ 3.2.1": version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -777,13 +777,6 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" -"glob@~ 3.2.1": - version "3.2.11" - resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" - dependencies: - inherits "2" - minimatch "0.3" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1227,13 +1220,6 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -minimatch@0.3: - version "0.3.0" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" - dependencies: - lru-cache "2" - sigmund "~1.0.0" - minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -1527,7 +1513,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2108,17 +2094,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#buffer-size: - version "2.3.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/9df4bde07f3cf9ffbda42657066559e2b8875e1b" +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": + version "2.3.1-cdb2" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#fix-buffer-size-0: - version "0.6.18-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/c67400573572e836609dd75a7acec8f414a50dbe" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": + version "0.6.18-cdb2" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2279,9 +2265,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#548-vector-buffer-size: - version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" +windshaft@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.0.tgz#c69f8d199bbeccc96e66b8256928e6fa87999803" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2298,8 +2284,8 @@ windshaft@cartodb/windshaft#548-vector-buffer-size: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#buffer-size - tilelive-mapnik cartodb/tilelive-mapnik#fix-buffer-size-0 + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb2 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2 torque.js "~2.11.0" underscore "~1.6.0" From 8f156b9f134086112413a22d0ac1cb7f5c78be39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 13:23:22 +0200 Subject: [PATCH 113/402] Release 3.7.0 --- NEWS.md | 8 ++++++-- package.json | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index ca420d85..c0c8d7c0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,11 @@ # Changelog -## 3.6.7 -Released 2017-mm-dd +## 3.7.0 +Released 2017-05-18 + +Announcements: +- Manage multiple values of buffer-size for different formats +- Upgrades windshaft to [3.2.0](https://github.com/CartoDB/windshaft/releases/tag/3.2.0). ## 3.6.6 diff --git a/package.json b/package.json index d33bb526..e9a35d7c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.6.7", + "version": "3.7.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 4e715f6ba43140d7519406491473de5e5633a325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 13:24:48 +0200 Subject: [PATCH 114/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index c0c8d7c0..5c574b72 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.7.1 +Released 2017-mm-dd + + ## 3.7.0 Released 2017-05-18 diff --git a/package.json b/package.json index e9a35d7c..b066affd 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.7.0", + "version": "3.7.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From e9112da3053adcd10b05c2c0a0935a057377381d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 15:41:41 +0200 Subject: [PATCH 115/402] Going red: add test to check undefined buffersize in requested mapconfig throws error --- test/acceptance/buffer-size-format.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index d5157585..15be2e66 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -283,7 +283,7 @@ describe('buffer size per format for named maps', function () { }); -describe('buffer size per format for named maps w/o placeholders', function () { +describe.only('buffer size per format for named maps w/o placeholders', function () { var testCases = [ { desc: 'should get png tile using buffer-size 0 overriden by template params', @@ -396,7 +396,23 @@ describe('buffer size per format for named maps w/o placeholders', function () { assert: function (tile, callback) { assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); } - } + }, + { + desc: 'should get png tile using buffer-size 0 overriden by template params with no buffersize in mapconfig', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { + buffersize: { + png: 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-no-buffer-size-mapconfig-png-0', undefined, {}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + ]; testCases.forEach(function (test) { From 3ce38d7081e002b28d9edcbf7974f1f7a62d614f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 15:49:57 +0200 Subject: [PATCH 116/402] Going green: fix type error when no buffersize is defined in mapconfig --- .../mapconfig/adapter/mapconfig-buffer-size-adapter.js | 5 +++++ test/acceptance/buffer-size-format.js | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js index 3296d0ea..7e9e93a1 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -9,8 +9,13 @@ MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapCo return callback(null, requestMapConfig); } + this.formats.forEach(function (format) { if (Number.isFinite(context.templateParams.buffersize[format])) { + if (requestMapConfig.buffersize === undefined) { + requestMapConfig.buffersize = {}; + } + requestMapConfig.buffersize[format] = context.templateParams.buffersize[format]; } }); diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 15be2e66..c32f62d1 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -283,7 +283,7 @@ describe('buffer size per format for named maps', function () { }); -describe.only('buffer size per format for named maps w/o placeholders', function () { +describe('buffer size per format for named maps w/o placeholders', function () { var testCases = [ { desc: 'should get png tile using buffer-size 0 overriden by template params', @@ -398,7 +398,8 @@ describe.only('buffer size per format for named maps w/o placeholders', function } }, { - desc: 'should get png tile using buffer-size 0 overriden by template params with no buffersize in mapconfig', + desc: 'should get png tile using buffer-size 0' + + ' overriden by template params with no buffersize in mapconfig', coords: { z: 7, x: 64, y: 48 }, format: 'png', placeholders: { From 3978d58d6669a2750e42c7d3414003bbfc4a0190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 16:12:28 +0200 Subject: [PATCH 117/402] Remove empty line --- .../models/mapconfig/adapter/mapconfig-buffer-size-adapter.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js index 7e9e93a1..aead2d91 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -9,7 +9,6 @@ MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapCo return callback(null, requestMapConfig); } - this.formats.forEach(function (format) { if (Number.isFinite(context.templateParams.buffersize[format])) { if (requestMapConfig.buffersize === undefined) { From 67aa2d1a00f9d690a903a5356bab007cdad992be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 16:34:56 +0200 Subject: [PATCH 118/402] Release 3.7.1 --- NEWS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5c574b72..4c501f91 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,10 @@ # Changelog ## 3.7.1 -Released 2017-mm-dd +Released 2017-05-18 + +Bug fixes: + - Fix buffersize assignment when is not defined in requested mapconfig. ## 3.7.0 From 61b19856e98de9d3f8ed94a1f1fa3b213068f4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 18 May 2017 16:36:50 +0200 Subject: [PATCH 119/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 4c501f91..88f3f616 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.7.2 +Released 2017-mm-dd + + ## 3.7.1 Released 2017-05-18 diff --git a/package.json b/package.json index b066affd..20c450ef 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.7.1", + "version": "3.7.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 45c1ccab9e4fbd8eaa072f80be5989ec974e3b07 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 18 May 2017 17:39:26 +0200 Subject: [PATCH 120/402] Upgrade mocha dep --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 20c450ef..764bf54a 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "devDependencies": { "istanbul": "~0.4.3", "jshint": "~2.6.0", - "mocha": "~1.21.4", + "mocha": "~3.4.1", "nock": "~2.11.0", "redis": "~0.12.1", "semver": "~1.1.4", From 0a3d1fbdf9a746127709af014c974cd383c41261 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 18 May 2017 17:39:56 +0200 Subject: [PATCH 121/402] Upgrade jshint and fix test --- package.json | 2 +- test/acceptance/ported/server_png8_format.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 764bf54a..488f7e88 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ }, "devDependencies": { "istanbul": "~0.4.3", - "jshint": "~2.6.0", + "jshint": "~2.9.4", "mocha": "~3.4.1", "nock": "~2.11.0", "redis": "~0.12.1", diff --git a/test/acceptance/ported/server_png8_format.js b/test/acceptance/ported/server_png8_format.js index 30b5f2bc..a710cbd0 100644 --- a/test/acceptance/ported/server_png8_format.js +++ b/test/acceptance/ported/server_png8_format.js @@ -16,13 +16,13 @@ describe('server_png8_format', function() { var serverOptionsPng32 = ServerOptions; serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore); serverOptionsPng32.grainstore.mapnik_tile_format = 'png32'; - var serverPng32 = new cartodbServer(serverOptionsPng32); + var serverPng32 = cartodbServer(serverOptionsPng32); serverPng32.setMaxListeners(0); var serverOptionsPng8 = ServerOptions; serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore); serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h'; - var serverPng8 = new cartodbServer(serverOptionsPng8); + var serverPng8 = cartodbServer(serverOptionsPng8); serverPng8.setMaxListeners(0); From 7efb196247177a972dc2b3fcb02f2c71c811342b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 22 May 2017 13:09:42 +0200 Subject: [PATCH 122/402] Upgrade camshaft to 0.55.0 --- NEWS.md | 5 +- package.json | 4 +- yarn.lock | 251 ++++++++++++++++++++++++++++----------------------- 3 files changed, 142 insertions(+), 118 deletions(-) diff --git a/NEWS.md b/NEWS.md index 88f3f616..a26dfbfc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,11 @@ # Changelog -## 3.7.2 +## 3.8.0 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0). + ## 3.7.1 Released 2017-05-18 diff --git a/package.json b/package.json index 488f7e88..34a41c77 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.7.2", + "version": "3.8.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.54.4", + "camshaft": "0.55.0", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index a203d58e..db83a7da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -157,6 +157,10 @@ brace-expansion@^1.0.0: balanced-match "^0.4.1" concat-map "0.0.1" +browser-stdout@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" + buffer-shims@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" @@ -194,9 +198,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.54.4: - version "0.54.4" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.54.4.tgz#3e1e96cda36661ebc01b64dadcc6387b02d69739" +camshaft@0.55.0: + version "0.55.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.0.tgz#dfe9d3638962e9e7f80a227db69071f3bbf3c315" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -205,7 +209,7 @@ camshaft@0.54.4: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +235,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -303,12 +307,12 @@ chroma-js@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.1.1.tgz#9bb9434959336ece75700aaadfeedc71806d8c05" -cli@0.6.x: - version "0.6.6" - resolved "https://registry.yarnpkg.com/cli/-/cli-0.6.6.tgz#02ad44a380abf27adac5e6f0cdd7b043d74c53e3" +cli@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cli/-/cli-1.0.1.tgz#22817534f24bfa4950c34d532d48ecbc621b8c14" dependencies: exit "0.1.2" - glob "~ 3.2.1" + glob "^7.1.1" cliui@^2.1.0: version "2.1.0" @@ -344,15 +348,7 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-0.6.1.tgz#fa68a14f6a945d54dbbe50d8cdb3320e9e3b1a06" - -commander@2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.3.0.tgz#fd430e889832ec353b9acd1de217c11cb3eef873" - -commander@^2.9.0: +commander@2.9.0, commander@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: @@ -412,18 +408,18 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.0.0.tgz#89bd9df6732b51256bc6705342bba02ed12131ef" - dependencies: - ms "0.6.2" - debug@2.2.0, debug@^2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: ms "0.7.1" +debug@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b" + dependencies: + ms "0.7.2" + debug@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" @@ -468,9 +464,9 @@ destroy@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" -diff@1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/diff/-/diff-1.0.8.tgz#343276308ec991b7bc82267ed55bc1411f971666" +diff@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9" dom-serializer@0: version "0.1.0" @@ -542,7 +538,11 @@ escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" -escape-string-regexp@1.0.2, escape-string-regexp@^1.0.2: +escape-string-regexp@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + +escape-string-regexp@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.2.tgz#4dbc2fe674e71949caf3fb2695ce7f2dc1d9a8d1" @@ -738,13 +738,16 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@3.2.3, "glob@~ 3.2.1": - version "3.2.3" - resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" +glob@7.1.1, glob@^7.0.5, glob@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: - graceful-fs "~2.0.0" + fs.realpath "^1.0.0" + inflight "^1.0.4" inherits "2" - minimatch "~0.2.11" + minimatch "^3.0.2" + once "^1.3.0" + path-is-absolute "^1.0.0" glob@^5.0.15: version "5.0.15" @@ -766,25 +769,10 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5: - version "7.1.1" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.2" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" -graceful-fs@~2.0.0: - version "2.0.3" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-2.0.3.tgz#7cd2cdb228a4a3f36e95efa6cc142de7d1a136d0" - "graceful-readlink@>= 1.0.0": version "1.0.1" resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" @@ -803,9 +791,9 @@ grainstore@~1.6.0: semver "~5.0.3" underscore "~1.6.0" -growl@1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/growl/-/growl-1.8.1.tgz#4b2dec8d907e93db336624dcec0183502f8c9428" +growl@1.9.2: + version "1.9.2" + resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: version "4.0.8" @@ -1007,13 +995,6 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" -jade@0.26.3: - version "0.26.3" - resolved "https://registry.yarnpkg.com/jade/-/jade-0.26.3.tgz#8f10d7977d8d79f2f6ff862a81b0513ccb25686c" - dependencies: - commander "0.6.1" - mkdirp "0.3.0" - jodid25519@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" @@ -1035,18 +1016,18 @@ jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" -jshint@~2.6.0: - version "2.6.3" - resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.6.3.tgz#84b470b8e5d5cd7adf0a3bd4975250443c9d311a" +jshint@~2.9.4: + version "2.9.4" + resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.4.tgz#5e3ba97848d5290273db514aee47fe24cf592934" dependencies: - cli "0.6.x" + cli "~1.0.0" console-browserify "1.1.x" exit "0.1.x" htmlparser2 "3.8.x" - minimatch "1.0.x" + lodash "3.7.x" + minimatch "~3.0.2" shelljs "0.3.x" strip-json-comments "1.0.x" - underscore "1.6.x" json-schema@0.2.3: version "0.2.3" @@ -1062,6 +1043,10 @@ json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" +json3@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/json3/-/json3-3.3.2.tgz#3c0434743df93e2f5c42aee7b19bcb483575f4e1" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -1112,14 +1097,65 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" +lodash._baseassign@^3.0.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz#8c38a099500f215ad09e59f1722fd0c52bfe0a4e" + dependencies: + lodash._basecopy "^3.0.0" + lodash.keys "^3.0.0" + +lodash._basecopy@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz#8da0e6a876cf344c0ad8a54882111dd3c5c7ca36" + +lodash._basecreate@^3.0.0: + version "3.0.3" + resolved "https://registry.yarnpkg.com/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz#1bc661614daa7fc311b7d03bf16806a0213cf821" + +lodash._getnative@^3.0.0: + version "3.9.1" + resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" + +lodash._isiterateecall@^3.0.0: + version "3.0.9" + resolved "https://registry.yarnpkg.com/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz#5203ad7ba425fae842460e696db9cf3e6aac057c" + lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.1.0, lodash.assign@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" +lodash.create@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/lodash.create/-/lodash.create-3.1.1.tgz#d7f2849f0dbda7e04682bb8cd72ab022461debe7" + dependencies: + lodash._baseassign "^3.0.0" + lodash._basecreate "^3.0.0" + lodash._isiterateecall "^3.0.0" + +lodash.isarguments@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + +lodash.isarray@^3.0.0: + version "3.0.4" + resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55" + +lodash.keys@^3.0.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a" + dependencies: + lodash._getnative "^3.0.0" + lodash.isarguments "^3.0.0" + lodash.isarray "^3.0.0" + lodash@2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.1.tgz#5b7723034dda4d262e5a46fb2c58d7cc22f71420" +lodash@3.7.x: + version "3.7.0" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.7.0.tgz#3678bd8ab995057c07ade836ed2ef087da811d45" + lodash@^4.5.1: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -1137,7 +1173,7 @@ longest@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" -lru-cache@2, lru-cache@2.6.5: +lru-cache@2.6.5: version "2.6.5" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.6.5.tgz#e56d6354148ede8d7707b58d143220fd08df0fd5" @@ -1220,26 +1256,12 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -minimatch@1.0.x: - version "1.0.0" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" - dependencies: - lru-cache "2" - sigmund "~1.0.0" - -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" dependencies: brace-expansion "^1.0.0" -minimatch@~0.2.11: - version "0.2.14" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.2.14.tgz#c74e780574f63c6f9a090e90efbe6ef53a6a756a" - dependencies: - lru-cache "2" - sigmund "~1.0.0" - minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1252,34 +1274,27 @@ minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" -mkdirp@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e" - -mkdirp@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.0.tgz#1d73076a6df986cd9344e15e71fcc05a4c9abf12" - dependencies: - minimist "0.0.8" - -mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: +mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: minimist "0.0.8" -mocha@~1.21.4: - version "1.21.5" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-1.21.5.tgz#7c58b09174df976e434a23b1e8d639873fc529e9" +mocha@~3.4.1: + version "3.4.1" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.1.tgz#a3802b4aa381934cacb38de70cf771621da8f9af" dependencies: - commander "2.3.0" - debug "2.0.0" - diff "1.0.8" - escape-string-regexp "1.0.2" - glob "3.2.3" - growl "1.8.1" - jade "0.26.3" - mkdirp "0.5.0" + browser-stdout "1.3.0" + commander "2.9.0" + debug "2.6.0" + diff "3.2.0" + escape-string-regexp "1.0.5" + glob "7.1.1" + growl "1.9.2" + json3 "3.3.2" + lodash.create "3.1.1" + mkdirp "0.5.1" + supports-color "3.1.2" moment@^2.10.6: version "2.18.1" @@ -1293,6 +1308,10 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" +ms@0.7.2: + version "0.7.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1734,7 +1753,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1759,7 +1778,7 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.81.0: +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1894,10 +1913,6 @@ shelljs@0.3.x: version "0.3.0" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" -sigmund@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590" - signal-exit@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" @@ -2052,6 +2067,12 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" +supports-color@3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5" + dependencies: + has-flag "^1.0.0" + supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" @@ -2094,7 +2115,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2102,7 +2123,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2196,10 +2217,6 @@ uid-number@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" -underscore@1.6.x, underscore@~1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" - underscore@1.8.2: version "1.8.2" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.2.tgz#64df2eb590899de950782f3735190ba42ebf311d" @@ -2208,6 +2225,10 @@ underscore@1.8.3: version "1.8.3" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" +underscore@~1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" + unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" From 76c2f8dc0e7621c316df4e309484df1429ca5592 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 22 May 2017 14:33:14 +0200 Subject: [PATCH 123/402] Includes release changes of turbo-carto too --- NEWS.md | 1 + package.json | 2 +- yarn.lock | 16 ++++++---------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index a26dfbfc..6ee1f28a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ Released 2017-mm-dd Announcements: - Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0). + - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1) ## 3.7.1 diff --git a/package.json b/package.json index 34a41c77..9df048bb 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "request": "~2.79.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.19.0", + "turbo-carto": "0.19.1", "underscore": "~1.6.0", "windshaft": "3.2.0", "yargs": "~5.0.0" diff --git a/yarn.lock b/yarn.lock index db83a7da..154a3478 100644 --- a/yarn.lock +++ b/yarn.lock @@ -538,14 +538,10 @@ escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" -escape-string-regexp@1.0.5: +escape-string-regexp@1.0.5, escape-string-regexp@^1.0.2: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" -escape-string-regexp@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.2.tgz#4dbc2fe674e71949caf3fb2695ce7f2dc1d9a8d1" - escodegen@1.8.x: version "1.8.1" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.8.1.tgz#5a5b53af4693110bebb0867aa3430dd3b70a1018" @@ -1753,7 +1749,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1778,7 +1774,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -2164,9 +2160,9 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@0.19.0: - version "0.19.0" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.0.tgz#83fb1932acd42acb426312eef216b5f6ac34708e" +turbo-carto@0.19.1: + version "0.19.1" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.1.tgz#c32af073936a4e8f197dfea918e7441c949d7865" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" From dcb26560e3fd9d2e4eaff26dd7a561026d721235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 22 May 2017 14:50:41 +0200 Subject: [PATCH 124/402] Release 3.8.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6ee1f28a..6a0ae8cd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.8.0 -Released 2017-mm-dd +Released 2017-05-22 Announcements: - Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0). From 5135b6e14ac32d7fb08d05c2f046d748f39355e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 22 May 2017 14:57:03 +0200 Subject: [PATCH 125/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6a0ae8cd..84e8a5d2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.8.1 +Released 2017-mm-dd + + ## 3.8.0 Released 2017-05-22 diff --git a/package.json b/package.json index 9df048bb..4eae5323 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.8.0", + "version": "3.8.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 8b0964ad7ecffa813f7efaa15d981cf491ac1f82 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 22 May 2017 18:45:24 +0200 Subject: [PATCH 126/402] Remove Promise hack from turbo-carto adapter --- .../models/mapconfig/adapter/turbo-carto-adapter.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index 46efd100..b9487095 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -4,13 +4,6 @@ var dot = require('dot'); dot.templateSettings.strip = false; var queue = require('queue-async'); var PSQL = require('cartodb-psql'); -/** - * cartodb-psql creates `global.Promise` as an empty constructor. - * However, `turbo-carto` relies on a polyfil that fails to create the polyfil - * as it finds `global.Promise` but it doesn't find `Promise.resolve`. - */ -global.Promise = global.Promise || function() {}; -global.Promise.resolve = global.Promise.resolve || function() {}; var turboCarto = require('turbo-carto'); var SubstitutionTokens = require('../../../utils/substitution-tokens'); From 51c11aff03676a33ce1188aa00527a36ed24cc77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 23 May 2017 13:37:34 +0200 Subject: [PATCH 127/402] Upgrade camshaft to 0.55.1 --- NEWS.md | 3 +++ package.json | 2 +- yarn.lock | 10 +++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 84e8a5d2..b68bf3ae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.8.1 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.55.1](https://github.com/CartoDB/camshaft/releases/tag/0.55.1). + ## 3.8.0 Released 2017-05-22 diff --git a/package.json b/package.json index 4eae5323..efca7cb0 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.0", + "camshaft": "0.55.1", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 154a3478..247eea09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -198,9 +198,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.0: - version "0.55.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.0.tgz#dfe9d3638962e9e7f80a227db69071f3bbf3c315" +camshaft@0.55.1: + version "0.55.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.1.tgz#548b286c311ed3a42933dcced071fb371c6993b9" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -1749,7 +1749,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1774,7 +1774,7 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.81.0: +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: From e5223980cf88448bef43cb767116e6e5b5546ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 23 May 2017 13:46:56 +0200 Subject: [PATCH 128/402] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index b68bf3ae..b8e12ce5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ Released 2017-mm-dd Announcements: - Upgrades camshaft to [0.55.1](https://github.com/CartoDB/camshaft/releases/tag/0.55.1). + - Remove promise polyfill from turbo-carto adapter ## 3.8.0 From 69984ed8953c28a0162758439af0d0d53d59fc41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 24 May 2017 11:54:30 +0200 Subject: [PATCH 129/402] Upgrade camshaft to 0.55.2 --- NEWS.md | 2 +- package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index b8e12ce5..446181ef 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,7 +4,7 @@ Released 2017-mm-dd Announcements: - - Upgrades camshaft to [0.55.1](https://github.com/CartoDB/camshaft/releases/tag/0.55.1). + - Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2). - Remove promise polyfill from turbo-carto adapter diff --git a/package.json b/package.json index efca7cb0..3a9f9770 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.1", + "camshaft": "0.55.2", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 247eea09..f2c7cfd6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -198,9 +198,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.1: - version "0.55.1" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.1.tgz#548b286c311ed3a42933dcced071fb371c6993b9" +camshaft@0.55.2: + version "0.55.2" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.2.tgz#ee756633b7cde16891874c3441b48b5178ff55a2" dependencies: async "^1.5.2" bunyan "1.8.1" From b84bb469e50f49f8e8c3947ac6b323a1ddab2fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 25 May 2017 11:26:04 +0200 Subject: [PATCH 130/402] Allow es6 syntax --- .jshintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jshintrc b/.jshintrc index 00a59a27..fbc4c96e 100644 --- a/.jshintrc +++ b/.jshintrc @@ -40,7 +40,7 @@ "debug" : false, // true: Allow debugger statements e.g. browser breakpoints. // "eqnull" : false, // true: Tolerate use of `== null` // "es5" : false, // true: Allow ES5 syntax (ex: getters and setters) -// "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`) + "esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`) // "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features) // // (ex: `for each`, multiple try/catch, function expression…) // "evil" : false, // true: Tolerate use of `eval` and `new Function()` From f4e99629f613c52df8c079086ee7fe0c7c235f4e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 26 May 2017 13:02:02 +0200 Subject: [PATCH 131/402] Do not assert inside response, but pass error into callback Preferably we should put response outside of assert and change its callback signature. However, I don't think it is worth the effort right now. --- test/support/assert.js | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/test/support/assert.js b/test/support/assert.js index a53d6fbf..c5e22e93 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -126,22 +126,25 @@ assert.response = function(server, req, res, callback) { // Assert response body if (res.body) { var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body; - assert.ok( - eql, - colorize('[red]{Invalid response body.}\n' + + if (!eql) { + return callback(response, new Error(colorize( + '[red]{Invalid response body.}\n' + ' Expected: [green]{' + res.body + '}\n' + - ' Got: [red]{' + response.body + '}') - ); + ' Got: [red]{' + response.body + '}')) + ); + } } // Assert response status if (typeof status === 'number') { - assert.equal(response.statusCode, status, - colorize('[red]{Invalid response status code.}\n' + + if (response.statusCode != status) { + return callback(response, new Error(colorize( + '[red]{Invalid response status code.}\n' + ' Expected: [green]{' + status + '}\n' + ' Got: [red]{' + response.statusCode + '}\n' + - ' Body: ' + response.body) - ); + ' Body: ' + response.body)) + ); + } } // Assert response headers @@ -152,11 +155,13 @@ assert.response = function(server, req, res, callback) { actual = response.headers[name.toLowerCase()], expected = res.headers[name], headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual; - assert.ok(headerEql, - colorize('Invalid response header [bold]{' + name + '}.\n' + + if (!headerEql) { + return callback(response, new Error(colorize( + 'Invalid response header [bold]{' + name + '}.\n' + ' Expected: [green]{' + expected + '}\n' + - ' Got: [red]{' + actual + '}') - ); + ' Got: [red]{' + actual + '}')) + ); + } } } From 248adab05ba4261478ba01b96ee61a31c87f3f16 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 26 May 2017 13:06:04 +0200 Subject: [PATCH 132/402] Catch response body if any to capture Redis keys --- test/support/test-client.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index bbb32c9d..04980a5a 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -617,18 +617,20 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { }, expectedResponse, function(res, err) { + // If there is a response, we are still interested in catching the created keys + // to be able to delete them on the .drain() method. + if (res) { + var parsedBody = JSON.parse(res.body); + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + } if (err) { return callback(err); } - var parsedBody = JSON.parse(res.body); - - if (parsedBody.layergroupid) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - } - - return callback(null, parsedBody); + return callback(null, JSON.parse(res.body)); } ); }; From 882aeacac23204f6b79c38e3fe6738073f05f8d7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 26 May 2017 13:13:19 +0200 Subject: [PATCH 133/402] Rewrite test to take advantage of changes in assert.response/TestClient This should avoid the issue of preventing the whole suite to halt, as in https://travis-ci.org/CartoDB/Windshaft-cartodb/builds/236337027. --- .../ported/multilayer_error_cases.js | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index f0464699..3408baa2 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -5,6 +5,7 @@ var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); +var TestClient = require('../../support/test-client'); var BaseController = require('../../../lib/cartodb/controllers/base'); @@ -23,6 +24,14 @@ describe('multilayer error cases', function() { BaseController.prototype.req2params = req2paramsFn; }); + // var client = null; + afterEach(function(done) { + if (this.client) { + return this.client.drain(done); + } + return done(); + }); + it("post layergroup with wrong Content-Type", function(done) { assert.response(server, { url: '/database/windshaft_test/layergroup', @@ -153,24 +162,16 @@ describe('multilayer error cases', function() { ] }; ServerOptions.afterLayergroupCreateCalls = 0; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(layergroup) - }, {}, function(res) { - try { - assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); - // See http://github.com/CartoDB/Windshaft/issues/159 - assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); - var parsed = JSON.parse(res.body); - assert.ok(parsed); - assert.equal(parsed.errors.length, 1); - var error = parsed.errors[0]; - assert.ok(error.match(/column "missing" does not exist/m), error); - // TODO: check which layer introduced the problem ? - done(); - } catch (err) { done(err); } + this.client = new TestClient(layergroup); + this.client.getLayergroup({status: 400}, function(err, parsed) { + assert.ok(!err, err); + // See http://github.com/CartoDB/Windshaft/issues/159 + assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); + assert.ok(parsed); + assert.equal(parsed.errors.length, 1); + var error = parsed.errors[0]; + assert.ok(error.match(/column "missing" does not exist/m), error); + done(); }); }); From 440953b1cd6a81c7e8923edf3bfabd6756ce74ce Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 30 May 2017 10:47:47 +0200 Subject: [PATCH 134/402] If we have the stats FF disbabled return empty array instead of null But we keep checking for elements in the returned object because we don't want to include the stats property if the FF is disabled --- lib/cartodb/backends/stats.js | 2 +- lib/cartodb/controllers/map.js | 2 +- .../stats/mapnik_stats_layergroup.js | 27 ++++++++++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 20f89a0f..b0385bac 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -11,6 +11,6 @@ StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { if (layerStatsEnabled) { layerStats().getStats(mapConfig, dbConnection, callback); } else { - return callback(null, null); + return callback(null, []); } }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 3bb72574..282c3145 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -368,7 +368,7 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) { if (err) { return next(err); } - if (layersStats) { + if (layersStats.length > 0) { layergroup.metadata.layers.forEach(function (layer, index) { layer.meta.stats = layersStats[index]; }); diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 77288d51..cdac7bb1 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -5,14 +5,11 @@ var TestClient = require('../../support/test-client'); describe('Create mapnik layergroup', function() { before(function() { - this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; this.layerStatsConfig = global.environment.enabledFeatures.layerStats; - global.environment.enabledFeatures.layerMetadata = true; global.environment.enabledFeatures.layerStats = true; }); after(function() { - global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); @@ -255,4 +252,28 @@ describe('Create mapnik layergroup', function() { testClient.drain(done); }); }); + + it('should not include the stats part if the FF is disabled', function(done) { + global.environment.enabledFeatures.layerStats = false; + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + httpLayer, + mapnikLayer1, + httpLayer + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[0].type, 'http'); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); + assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats')); + assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1)); + assert.equal(layergroup.metadata.layers[2].type, 'http'); + testClient.drain(done); + }); + }); }); From da3f30dd9f121d57cc3b86192cdd1db16d4af2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 31 May 2017 11:27:41 +0200 Subject: [PATCH 135/402] Update NEWS --- NEWS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b8e12ce5..744e524c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,10 @@ # Changelog -## 3.8.1 +## 3.9.0 Released 2017-mm-dd Announcements: + - Add support to retrieve info about layer stats in map instantiation. - Upgrades camshaft to [0.55.1](https://github.com/CartoDB/camshaft/releases/tag/0.55.1). - Remove promise polyfill from turbo-carto adapter From 2c509d97b1532ad514b4664eda96c2842cf9d42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 31 May 2017 11:46:46 +0200 Subject: [PATCH 136/402] Point windshaft to the last released version --- NEWS.md | 1 + package.json | 4 ++-- yarn.lock | 38 +++++++++++++++++++------------------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/NEWS.md b/NEWS.md index 744e524c..e0de3ca5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,7 @@ Released 2017-mm-dd Announcements: + - Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1). - Add support to retrieve info about layer stats in map instantiation. - Upgrades camshaft to [0.55.1](https://github.com/CartoDB/camshaft/releases/tag/0.55.1). - Remove promise polyfill from turbo-carto adapter diff --git a/package.json b/package.json index 48db22e6..f8f5a56c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.8.1", + "version": "3.9.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.1", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#9624bb093cb2f7056ddcbacdfa5f4c276ed79edb", + "windshaft": "3.2.1", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 6d05a362..158b2529 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -209,7 +209,7 @@ camshaft@0.55.1: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -227,7 +227,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@cartodb/carto#0.15.1-cdb3: +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -235,14 +243,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -carto@cartodb/carto#0.15.1-cdb3: - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -1256,7 +1256,7 @@ mime@~1.2.11: version "3.0.3" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" dependencies: - brace-expansion "^1.1.7" + brace-expansion "^1.0.0" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1524,7 +1524,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1745,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1770,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -2107,7 +2107,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2115,7 +2115,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2278,9 +2278,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#9624bb093cb2f7056ddcbacdfa5f4c276ed79edb: +windshaft@3.2.1: version "3.2.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/9624bb093cb2f7056ddcbacdfa5f4c276ed79edb" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.1.tgz#50a3afa6562315dd9e65e411660970e118f36c19" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From dbd0398d9b45b432427ad131150ae6f24dfda064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 31 May 2017 12:13:25 +0200 Subject: [PATCH 137/402] Release 3.9.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5095b300..5886f65a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.9.0 -Released 2017-mm-dd +Released 2017-05-31 Announcements: - Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1). From c8ea595f47573a7e92784138eeed9171b3daab2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 31 May 2017 12:14:51 +0200 Subject: [PATCH 138/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5886f65a..930001d8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.1 +Released 2017-mm-dd + + ## 3.9.0 Released 2017-05-31 diff --git a/package.json b/package.json index 2690c8e5..230ca8ff 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.0", + "version": "3.9.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 31557b06beeadd6f85e3075400074b4376dcfa8f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 31 May 2017 11:48:06 +0200 Subject: [PATCH 139/402] Add test to detect and fix incorrect bbox filter splitting When bbox crosses date line and is split in two, the eastern box wasn't correct --- lib/cartodb/models/filter/bbox.js | 3 ++- .../cartodb/model/filter/bbox-filters.test.js | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js index 8afdf905..dcf19e55 100644 --- a/lib/cartodb/models/filter/bbox.js +++ b/lib/cartodb/models/filter/bbox.js @@ -66,7 +66,8 @@ function getBoundingBoxes(west, south, east, north) { bboxes.push([west, south, east, north]); } else { bboxes.push([west, south, 180, north]); - bboxes.push([-180, south, east % 180, north]); + // here we assume west,east have been adjusted => west >= -180 => east > 180 + bboxes.push([-180, south, east - 360, north]); } return bboxes; diff --git a/test/unit/cartodb/model/filter/bbox-filters.test.js b/test/unit/cartodb/model/filter/bbox-filters.test.js index df894892..68327a7a 100644 --- a/test/unit/cartodb/model/filter/bbox-filters.test.js +++ b/test/unit/cartodb/model/filter/bbox-filters.test.js @@ -111,6 +111,23 @@ describe('Bounding box filter', function() { createRef([-180, -45, 0, 45]) ); }); + + it('generating multiple bbox', function() { + var bbox = [90, -45, 190, 45]; + var bboxFilter = createFilter(bbox); + + assert.equal(bboxFilter.bboxes.length, 2); + + assert.deepEqual( + bboxFilter.bboxes[0], + createRef([90, -45, 180, 45]) + ); + assert.deepEqual( + bboxFilter.bboxes[1], + createRef([-180, -45, -170, 45]) + ); + }); + }); describe('out of bounds', function() { From 4e53803b3b3699926244724d550f16d4e8c7cb4f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 31 May 2017 12:51:07 -0400 Subject: [PATCH 140/402] Create BBoxFilter independently from rewrite data --- lib/cartodb/backends/dataview.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index e0515efe..f2e9c7df 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -43,6 +43,12 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ownFilter = !!ownFilter; var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; + var bboxFilter = null; + if (params.bbox) { + bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + var sourceId = dataviewDefinition.source.id; // node.id var layer = _.find(mapConfig.obj().layers, function(l) { return l.options.source && (l.options.source.id === sourceId); @@ -56,8 +62,6 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param } if (params.bbox) { - var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); - query = bboxFilter.sql(query); if ( queryRewriteData ) { var bbox_filter_definition = { type: 'bbox', From ef3cad65995e1eb70a84d1f5abd3f3f4ace4446e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 31 May 2017 12:52:41 -0400 Subject: [PATCH 141/402] Extract to function --- lib/cartodb/backends/dataview.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index f2e9c7df..82b8f438 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -81,16 +81,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } ); - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), - function castNumbers(overrides, val, k) { - if (!Number.isFinite(+val)) { - throw new Error('Invalid number format for parameter \'' + k + '\''); - } - overrides[k] = +val; - return overrides; - }, - {ownFilter: ownFilter} - ); + var overrideParams = getOverrideParams(params, ownFilter); var dataview = dataviewFactory.getDataview(query, dataviewDefinition); dataview.getResult(pg, overrideParams, this); @@ -101,6 +92,19 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ); }; +function getOverrideParams(params, ownFilter) { + return _.reduce(_.pick(params, 'start', 'end', 'bins'), + function castNumbers(overrides, val, k) { + if (!Number.isFinite(+val)) { + throw new Error('Invalid number format for parameter \'' + k + '\''); + } + overrides[k] = +val; + return overrides; + }, + {ownFilter: ownFilter} + ); +} + DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { var dataviewName = params.dataviewName; From 62514fc563e6f4da476ef0c58d39e3fd36717e4f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 31 May 2017 12:57:38 -0400 Subject: [PATCH 142/402] Extract query rewrite data to function --- lib/cartodb/backends/dataview.js | 64 +++++++++++++++++--------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 82b8f438..18ff2fb2 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -49,42 +49,14 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param query = bboxFilter.sql(query); } - var sourceId = dataviewDefinition.source.id; // node.id - var layer = _.find(mapConfig.obj().layers, function(l) { - return l.options.source && (l.options.source.id === sourceId); - }); - var queryRewriteData = layer && layer.options.query_rewrite_data; - if (queryRewriteData && dataviewDefinition.node.type === 'source') { - queryRewriteData = _.extend({}, queryRewriteData, { - filters: dataviewDefinition.node.filters, - unfiltered_query: dataviewDefinition.sql.own_filter_on - }); - } - - if (params.bbox) { - if ( queryRewriteData ) { - var bbox_filter_definition = { - type: 'bbox', - options: { - column: 'the_geom_webmercator', - srid: 3857 - }, - params: { - bbox: params.bbox - } - }; - queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition }); - } - } + var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); var dataviewFactory = DataviewFactoryWithOverviews.getFactory( overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } ); - var overrideParams = getOverrideParams(params, ownFilter); - var dataview = dataviewFactory.getDataview(query, dataviewDefinition); - dataview.getResult(pg, overrideParams, this); + dataview.getResult(pg, getOverrideParams(params, ownFilter), this); }, function returnCallback(err, result) { return callback(err, result); @@ -92,6 +64,38 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ); }; +function getQueryRewriteData(mapConfig, dataviewDefinition, params) { + var sourceId = dataviewDefinition.source.id; // node.id + var layer = _.find(mapConfig.obj().layers, function(l) { + return l.options.source && (l.options.source.id === sourceId); + }); + var queryRewriteData = layer && layer.options.query_rewrite_data; + if (queryRewriteData && dataviewDefinition.node.type === 'source') { + queryRewriteData = _.extend({}, queryRewriteData, { + filters: dataviewDefinition.node.filters, + unfiltered_query: dataviewDefinition.sql.own_filter_on + }); + } + + if (params.bbox) { + if ( queryRewriteData ) { + var bbox_filter_definition = { + type: 'bbox', + options: { + column: 'the_geom_webmercator', + srid: 3857 + }, + params: { + bbox: params.bbox + } + }; + queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition }); + } + } + + return queryRewriteData; +} + function getOverrideParams(params, ownFilter) { return _.reduce(_.pick(params, 'start', 'end', 'bins'), function castNumbers(overrides, val, k) { From b4fbe0b8cffd5fa3360de7a803f4659742643975 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 31 May 2017 12:58:35 -0400 Subject: [PATCH 143/402] No in advance var definition --- lib/cartodb/backends/dataview.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 18ff2fb2..d4c40f79 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -43,9 +43,8 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ownFilter = !!ownFilter; var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; - var bboxFilter = null; if (params.bbox) { - bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); + var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); query = bboxFilter.sql(query); } From 4b562e6768f48e5e2621ae36180ce95f48026f20 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 31 May 2017 13:00:37 -0400 Subject: [PATCH 144/402] Merge conditions --- lib/cartodb/backends/dataview.js | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index d4c40f79..a77ba585 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -76,20 +76,18 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) { }); } - if (params.bbox) { - if ( queryRewriteData ) { - var bbox_filter_definition = { - type: 'bbox', - options: { - column: 'the_geom_webmercator', - srid: 3857 - }, - params: { - bbox: params.bbox - } - }; - queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition }); - } + if (params.bbox && queryRewriteData) { + var bbox_filter_definition = { + type: 'bbox', + options: { + column: 'the_geom_webmercator', + srid: 3857 + }, + params: { + bbox: params.bbox + } + }; + queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition }); } return queryRewriteData; From 1d66e49910376ceeb66efb044df651c8a52c4316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 1 Jun 2017 20:07:46 +0200 Subject: [PATCH 145/402] WIP implemented date histogram --- lib/cartodb/backends/dataview.js | 7 ++ lib/cartodb/controllers/base.js | 1 + lib/cartodb/models/dataview/histogram.js | 96 ++++++++++++++++++++- test/acceptance/dataviews/histogram.js | 102 +++++++++++++++++++++-- test/support/test-client.js | 4 +- 5 files changed, 195 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index e0515efe..84f0ccef 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,6 +20,8 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } +var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'year']; + module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { @@ -30,6 +32,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param mapConfigProvider.getMapConfig(this); }, function runDataviewQuery(err, mapConfig) { + /* jshint maxcomplexity: 7 */ assert.ifError(err); var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); @@ -88,6 +91,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {ownFilter: ownFilter} ); + if (params.aggregation && DATE_AGGREGATIONS.indexOf(params.aggregation) !== -1) { + overrideParams.aggregation = params.aggregation; + } + var dataview = dataviewFactory.getDataview(query, dataviewDefinition); dataview.getResult(pg, overrideParams, this); }, diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index ab6587c2..6ce49fa4 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -25,6 +25,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'start', // number 'end', // number 'column_type', // string + 'aggregation', //string // widgets search 'q' ]; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5d102bf5..6f3a2492 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -8,7 +8,7 @@ dot.templateSettings.strip = false; var columnTypeQueryTpl = dot.template( 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' ); -var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +// var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -71,7 +71,7 @@ var overrideBinsQueryTpl = dot.template([ var nullsQueryTpl = dot.template([ 'nulls AS (', ' SELECT', - ' count(*) AS nulls_count', + ' count(*) AS nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' @@ -97,6 +97,52 @@ var histogramQueryTpl = dot.template([ 'ORDER BY bin' ].join('\n')); +var dateBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' min(date_trunc(\'{{=it._aggregation}}\', {{=it._column}})) AS start_date,', + ' max({{=it._column}}) AS end_date,', + ' count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var dateBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT', + ' bins_array,', + ' ARRAY_LENGTH(bins_array, 1) AS bins_number', + ' FROM (', + ' SELECT', + ' ARRAY(', + ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' ) AS bins_array', + ' FROM basics', + ' ) _cdb_bins_array', + ')' +].join('\n')); + +var dateHistogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1', + ' END AS bin,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', + ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', + 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'ORDER BY bin' +].join('\n')); var TYPE = 'histogram'; @@ -118,6 +164,7 @@ function Histogram(query, options, queries) { this.queries = queries; this.column = options.column; this.bins = options.bins; + this.aggregation = options.aggregation; this._columnType = null; } @@ -163,7 +210,7 @@ Histogram.prototype.sql = function(psql, override, callback) { } if (this._columnType === 'date') { - _column = columnCastTpl({column: _column}); + return this._buildDateHistogramQuery(override, callback); } var _query = this.query; @@ -211,7 +258,6 @@ Histogram.prototype.sql = function(psql, override, callback) { } } - var histogramSql = [ "WITH", [ @@ -233,6 +279,48 @@ Histogram.prototype.sql = function(psql, override, callback) { return callback(null, histogramSql); }; +Histogram.prototype._buildDateHistogramQuery = function (override, callback) { + var _column = this.column; + var _query = this.query; + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + + var dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation + }); + + var dateBinsQuery = [ + dateBinsQueryTpl({ + _aggregation: _aggregation + }) + ].join(',\n'); + + var nullsQuery = nullsQueryTpl({ + _query: _query, + _column: _column + }); + + var dateHistogramQuery = dateHistogramQueryTpl({ + _query: _query, + _column: _column + }); + + var histogramSql = [ + "WITH", + [ + dateBasicsQuery, + dateBinsQuery, + nullsQuery + ].join(',\n'), + dateHistogramQuery + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index abbaef61..c581380a 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -3,6 +3,15 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + describe('histogram-dataview', function() { afterEach(function(done) { @@ -13,15 +22,6 @@ describe('histogram-dataview', function() { } }); - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - var mapConfig = createMapConfig( [ { @@ -94,6 +94,90 @@ describe('histogram-dataview', function() { assert.equal(res.errors.length, 1); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); + done(); + }); + }); + +}); + +describe('histogram-dataview for date column type', function() { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "date-histogram-source" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month' + } + } + }, + [ + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval", + ") date" + ].join(' ') + } + } + ] + ); + + it('should create a date histogram aggregated in months', function (done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override aggregation in weeks', function (done) { + var params = { + aggregation: 'week' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + done(); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 04980a5a..1913d018 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } @@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } params.placeholders = params.placeholders || {}; - + assert.response(server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), From 9c34428984565f94594903e2f9ccb20ec54d5b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 12:15:43 +0200 Subject: [PATCH 146/402] Allow override start and end params --- lib/cartodb/models/dataview/histogram.js | 52 ++++++++++++++++++++---- test/acceptance/dataviews/histogram.js | 23 +++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6f3a2492..dd3baf95 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -8,7 +8,7 @@ dot.templateSettings.strip = false; var columnTypeQueryTpl = dot.template( 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' ); -// var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -110,6 +110,28 @@ var dateBasicsQueryTpl = dot.template([ ')' ].join(' \n')); +var overrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', + ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join('\n')); + +var dateOverrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val,', + ' min({{=it._start}}) AS min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' min(date_trunc(\'{{=it._aggregation}}\', TO_TIMESTAMP({{=it._start}})::timestamp)) AS start_date,', + ' max(TO_TIMESTAMP({{=it._end}})::timestamp) AS end_date,', + ' count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', @@ -181,6 +203,7 @@ var DATE_OIDS = { }; Histogram.prototype.sql = function(psql, override, callback) { + // jshint maxcomplexity: 7 if (!callback) { callback = override; override = {}; @@ -209,10 +232,13 @@ Histogram.prototype.sql = function(psql, override, callback) { return null; } - if (this._columnType === 'date') { + if (this._columnType === 'date' && this.aggregation !== undefined) { return this._buildDateHistogramQuery(override, callback); } + if (this._columnType === 'date') { + _column = columnCastTpl({ column: this.column}); + } var _query = this.query; var basicsQuery, binsQuery; @@ -284,11 +310,23 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var dateBasicsQuery = dateBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation - }); + var dateBasicsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end')) { + dateBasicsQuery = dateOverrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _start: getBinStart(override), + _end: getBinEnd(override) + }); + } else { + dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation + }); + } var dateBinsQuery = [ dateBinsQueryTpl({ diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c581380a..c50abebb 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -156,6 +156,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -174,6 +175,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -181,4 +183,25 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should override start and end', function (done) { + var params = { + start: 1180659600, // 2007-06-01 00:00:00 + end: 1193792400 // 2007-10-31 01:00:00 + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 5); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + }); From 6374d2e4b64929ce4394ce0bfef4e42a8c3bccc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 12:17:55 +0200 Subject: [PATCH 147/402] Fix typo --- test/acceptance/dataviews/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c50abebb..294c9106 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -186,7 +186,7 @@ describe('histogram-dataview for date column type', function() { it('should override start and end', function (done) { var params = { - start: 1180659600, // 2007-06-01 00:00:00 + start: 1180659600, // 2007-06-01 01:00:00 end: 1193792400 // 2007-10-31 01:00:00 }; From c1fac13d6bedf8140a5b6ed0eb3451f1540c70c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 12:45:34 +0200 Subject: [PATCH 148/402] Be able to accept timezone parameter --- lib/cartodb/backends/dataview.js | 2 +- lib/cartodb/controllers/base.js | 1 + lib/cartodb/models/dataview/histogram.js | 13 +++---------- test/acceptance/dataviews/histogram.js | 18 ++++++++++++++++++ test/support/test-client.js | 2 +- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 84f0ccef..dc43f864 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -80,7 +80,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } ); - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'timezone'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 6ce49fa4..91fcaba7 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -26,6 +26,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'end', // number 'column_type', // string 'aggregation', //string + 'timezone', // number // widgets search 'q' ]; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dd3baf95..649bd0f2 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -110,15 +110,6 @@ var dateBasicsQueryTpl = dot.template([ ')' ].join(' \n')); -var overrideBasicsQueryTpl = dot.template([ - 'basics AS (', - ' SELECT', - ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', - ')' -].join('\n')); - var dateOverrideBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', @@ -156,13 +147,14 @@ var dateHistogramQueryTpl = dot.template([ ' THEN 0', ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1', ' END AS bin,', + ' date_part(\'epoch\', start_date)::numeric AS bin_start,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', 'ORDER BY bin' ].join('\n')); @@ -187,6 +179,7 @@ function Histogram(query, options, queries) { this.column = options.column; this.bins = options.bins; this.aggregation = options.aggregation; + this.timezone = options.timezone; this._columnType = null; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 294c9106..06fc6b80 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -204,4 +204,22 @@ describe('histogram-dataview for date column type', function() { }); }); + it.skip('should aggregate respecting timezone', function (done) { + var params = { + timezone: -7200 // GMT -2h + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 5); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 1913d018..dc36929e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end', 'aggregation'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'timezone'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } From 8bc38a375a4b0189814e4563c060c2788f557035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 18:37:49 +0200 Subject: [PATCH 149/402] Support timezone aggregation --- lib/cartodb/models/dataview/histogram.js | 51 +++++++++++++++++++----- test/acceptance/dataviews/histogram.js | 11 ++--- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 649bd0f2..499f32cd 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -103,8 +103,10 @@ var dateBasicsQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', - ' min(date_trunc(\'{{=it._aggregation}}\', {{=it._column}})) AS start_date,', - ' max({{=it._column}}) AS end_date,', + ' min(date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', + ' )) AS start_date,', + ' max({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -116,8 +118,11 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' max({{=it._end}}) AS max_val,', ' min({{=it._start}}) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', - ' min(date_trunc(\'{{=it._aggregation}}\', TO_TIMESTAMP({{=it._start}})::timestamp)) AS start_date,', - ' max(TO_TIMESTAMP({{=it._end}})::timestamp) AS end_date,', + ' min(date_trunc(', + ' \'{{=it._aggregation}}\',', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' )) AS start_date,', + ' max(TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -145,9 +150,16 @@ var dateHistogramQueryTpl = dot.template([ ' nulls_count,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1', + ' ELSE GREATEST(1, LEAST(', + ' WIDTH_BUCKET({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\', bins_array),', + ' bins_number', + ' )) - 1', ' END AS bin,', - ' date_part(\'epoch\', start_date)::numeric AS bin_start,', + ' date_part(\'epoch\', start_date)::numeric AS bins_start,', + ' min(date_part(', + ' \'epoch\', ', + ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', + ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', @@ -302,6 +314,7 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var _timezone = override && override.timezone ? override.timezone : this.timezone; var dateBasicsQuery; @@ -311,13 +324,15 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { _column: _column, _aggregation: _aggregation, _start: getBinStart(override), - _end: getBinEnd(override) + _end: getBinEnd(override), + _timezone: getTimezone(_timezone) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, - _aggregation: _aggregation + _aggregation: _aggregation, + _timezone: getTimezone(_timezone) }); } @@ -334,7 +349,9 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var dateHistogramQuery = dateHistogramQueryTpl({ _query: _query, - _column: _column + _column: _column, + _aggregation: _aggregation, + _timezone: getTimezone(_timezone) }); var histogramSql = [ @@ -371,7 +388,7 @@ Histogram.prototype.format = function(result, override) { binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'bins_start'); }); } @@ -414,6 +431,20 @@ function getWidth(override) { return width; } +function getTimezone(timezone) { + if (!timezone) { + return '0'; + } + + var timezoneInHours = Math.ceil(timezone / 3600); + + if (timezoneInHours >= 0) { + return '+' + timezoneInHours; + } else { + return '' + timezoneInHours; + } +} + Histogram.prototype.getType = function() { return TYPE; }; diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 06fc6b80..f5fab192 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -130,7 +130,8 @@ describe('histogram-dataview for date column type', function() { type: 'histogram', options: { column: 'd', - aggregation: 'month' + aggregation: 'month', + timezone: -7200 } } }, @@ -195,7 +196,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 5); + assert.equal(dataview.bins.length, 6); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -204,9 +205,9 @@ describe('histogram-dataview for date column type', function() { }); }); - it.skip('should aggregate respecting timezone', function (done) { + it('should aggregate respecting timezone', function (done) { var params = { - timezone: -7200 // GMT -2h + timezone: 7200 // GMT +2h }; this.testClient = new TestClient(mapConfig, 1234); @@ -214,7 +215,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 5); + assert.equal(dataview.bins.length, 15); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); From 32eeb57fce7710e7a8ab936f9c52b7da0bb06fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 19:00:26 +0200 Subject: [PATCH 150/402] Reduce complexity in function --- lib/cartodb/models/dataview/histogram.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 499f32cd..e14aa4d1 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -437,12 +437,7 @@ function getTimezone(timezone) { } var timezoneInHours = Math.ceil(timezone / 3600); - - if (timezoneInHours >= 0) { - return '+' + timezoneInHours; - } else { - return '' + timezoneInHours; - } + return '' + timezoneInHours; } Histogram.prototype.getType = function() { From 6ff3b33cde2874e5300c4727081676fb702f6cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:17:50 +0200 Subject: [PATCH 151/402] Removed bins_start as query output --- lib/cartodb/models/dataview/histogram.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index e14aa4d1..17921776 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -155,7 +155,6 @@ var dateHistogramQueryTpl = dot.template([ ' bins_number', ' )) - 1', ' END AS bin,', - ' date_part(\'epoch\', start_date)::numeric AS bins_start,', ' min(date_part(', ' \'epoch\', ', ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', @@ -385,7 +384,11 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; - binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; + binsStart = override.hasOwnProperty('start') ? + getBinStart(override) : + firstRow.hasOwnProperty('timestamp') ? + firstRow.timestamp : + firstRow.min; buckets = result.rows.map(function(row) { return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'bins_start'); From 71e9e62db07fa404052703f14decd7c5993b49c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:18:24 +0200 Subject: [PATCH 152/402] Improved histogram assertion with moment.js --- package.json | 1 + test/acceptance/dataviews/histogram.js | 19 +++++++++++++++-- yarn.lock | 28 +++++++++++++------------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 230ca8ff..fea8b67c 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "istanbul": "~0.4.3", "jshint": "~2.9.4", "mocha": "~3.4.1", + "moment": "~2.18.1", "nock": "~2.11.0", "redis": "~0.12.1", "semver": "~1.1.4", diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index f5fab192..2ca2ac6b 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -2,6 +2,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +var moment = require('moment'); function createMapConfig(layers, dataviews, analysis) { return { @@ -206,8 +207,10 @@ describe('histogram-dataview for date column type', function() { }); it('should aggregate respecting timezone', function (done) { + var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) var params = { - timezone: 7200 // GMT +2h + timezone: TIMEZONE_CEST_IN_SECONDS }; this.testClient = new TestClient(mapConfig, 1234); @@ -216,7 +219,19 @@ describe('histogram-dataview for date column type', function() { assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); assert.equal(dataview.bins.length, 15); - dataview.bins.forEach(function(bin) { + + var initialTimestamp = '2007-02-01T00:00:00+02:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp).utcOffset(TIMEZONE_CEST_IN_MINUTES).add(index, 'month').format(); + console.log(binTimestampExpected); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); diff --git a/yarn.lock b/yarn.lock index 089b43ad..4cc18ef4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -150,7 +150,7 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.1.7: +brace-expansion@^1.0.0: version "1.1.7" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" dependencies: @@ -209,7 +209,7 @@ camshaft@0.55.2: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -235,7 +235,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1292,7 +1292,7 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6: +moment@^2.10.6, moment@^2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -1835,22 +1835,22 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@~4.3.3: - version "4.3.6" - resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" -semver@^5.1.0, semver@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - semver@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540" +semver@~4.3.3: + version "4.3.6" + resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" + semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" @@ -2107,7 +2107,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2115,7 +2115,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: From dc844f8131ab3033feedcf0e1f81156c082aa4da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:23:53 +0200 Subject: [PATCH 153/402] Remove console.log --- test/acceptance/dataviews/histogram.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 2ca2ac6b..62a4aac8 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -227,11 +227,10 @@ describe('histogram-dataview for date column type', function() { dataview.bins.forEach(function(bin, index) { var binTimestampExpected = moment.utc(initialTimestamp).utcOffset(TIMEZONE_CEST_IN_MINUTES).add(index, 'month').format(); - console.log(binTimestampExpected); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); - assert.equal(binTimestampFormatted, binTimestampExpected); + assert.equal(binTimestampFormatted, binTimestampExpected); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); From 869408b7b7057641814e734d7e22e4b1e1e6f068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:50:49 +0200 Subject: [PATCH 154/402] Use Eastern Daylight Time while testing --- test/acceptance/dataviews/histogram.js | 39 ++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 62a4aac8..b2c10ea5 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -132,7 +132,7 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd', aggregation: 'month', - timezone: -7200 + timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } } }, @@ -153,13 +153,35 @@ describe('histogram-dataview for date column type', function() { ); it('should create a date histogram aggregated in months', function (done) { + var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', {}, function(err, dataview) { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); assert.equal(dataview.bins.length, 15); - dataview.bins.forEach(function(bin) { + + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min && 'bin timestamp < bin min: ' + JSON.stringify(bin)); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -222,13 +244,20 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00+02:00'; var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp).utcOffset(TIMEZONE_CEST_IN_MINUTES).add(index, 'month').format(); + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .add(index, 'month') + .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .format(); assert.equal(binTimestampFormatted, binTimestampExpected); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); From 003227fb295f927edaef9151308cde328ad92520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:59:35 +0200 Subject: [PATCH 155/402] Fix assertion --- test/acceptance/dataviews/histogram.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index b2c10ea5..adcab0e1 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -181,7 +181,7 @@ describe('histogram-dataview for date column type', function() { .format(); assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min && 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -260,6 +260,7 @@ describe('histogram-dataview for date column type', function() { .format(); assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); From c86f92f8eb49cc885c7422cd2c3b5db5ad5a344b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 15:04:40 +0200 Subject: [PATCH 156/402] Improve test description --- test/acceptance/dataviews/histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index adcab0e1..cc9e3dc1 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -152,7 +152,7 @@ describe('histogram-dataview for date column type', function() { ] ); - it('should create a date histogram aggregated in months', function (done) { + it('should create a date histogram aggregated in months (EDT)', function (done) { var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes this.testClient = new TestClient(mapConfig, 1234); @@ -228,7 +228,7 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should aggregate respecting timezone', function (done) { + it('should aggregate histogram overriding default timezone to CEST', function (done) { var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) var params = { @@ -242,7 +242,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); assert.equal(dataview.bins.length, 15); - var initialTimestamp = '2007-02-01T00:00:00+02:00'; + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_CEST_IN_MINUTES) From d01787842f93f404d28ccb12c26aa374ccd084ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 15:23:04 +0200 Subject: [PATCH 157/402] Support UTC timezone override --- lib/cartodb/models/dataview/histogram.js | 3 +- test/acceptance/dataviews/histogram.js | 40 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 17921776..1328383d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -313,7 +313,7 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _timezone = override && override.timezone ? override.timezone : this.timezone; + var _timezone = override && Number.isFinite(override.timezone) ? override.timezone : this.timezone; var dateBasicsQuery; @@ -438,7 +438,6 @@ function getTimezone(timezone) { if (!timezone) { return '0'; } - var timezoneInHours = Math.ceil(timezone / 3600); return '' + timezoneInHours; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index cc9e3dc1..5e4de9b9 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -267,4 +267,44 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should aggregate histogram overriding default timezone to UTC/GMT', function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); }); From 95a6ad3b86cea3b4f4a862b9a5d6249b389bba2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:04:42 +0200 Subject: [PATCH 158/402] Support quarter aggregation in histograms over date columns --- lib/cartodb/backends/dataview.js | 2 +- lib/cartodb/models/dataview/histogram.js | 4 +++ test/acceptance/dataviews/histogram.js | 42 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index dc43f864..ca451d38 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,7 +20,7 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } -var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'year']; +var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'quarter', 'year']; module.exports = DataviewBackend; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 1328383d..be4c7193 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -136,7 +136,11 @@ var dateBinsQueryTpl = dot.template([ ' FROM (', ' SELECT', ' ARRAY(', + ' {{?it._aggregation==="quarter"}}', + ' SELECT GENERATE_SERIES(start_date, end_date, \'3 month\'::interval)', + ' {{??}}', ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' {{?}}', ' ) AS bins_array', ' FROM basics', ' ) _cdb_bins_array', diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 5e4de9b9..0efb56d6 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -307,4 +307,46 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should aggregate histogram using "quarter" aggregation', function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + }); From e82d688a187c6624213a26bbb74e27438945c3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:09:47 +0200 Subject: [PATCH 159/402] Fix typo --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index be4c7193..9db9e0e7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -161,9 +161,9 @@ var dateHistogramQueryTpl = dot.template([ ' END AS bin,', ' min(date_part(', ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', + ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', - ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' min(date_part(\'epoch\', ))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', From 1ff095439014259ea89262319eed89909618a812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:25:47 +0200 Subject: [PATCH 160/402] Fix typo --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index be4c7193..f9162169 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -161,7 +161,7 @@ var dateHistogramQueryTpl = dot.template([ ' END AS bin,', ' min(date_part(', ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', + ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', From 9337cd948cf1b175bdbf038469574bf17e769d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:26:29 +0200 Subject: [PATCH 161/402] Fix typo --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 9db9e0e7..f9162169 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -163,7 +163,7 @@ var dateHistogramQueryTpl = dot.template([ ' \'epoch\', ', ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', - ' min(date_part(\'epoch\', ))::numeric AS min,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', From eeea51e10dece2e9d7c99b16b3064373d2661f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 17:26:37 +0200 Subject: [PATCH 162/402] Removed unused column in group by statement --- lib/cartodb/models/dataview/histogram.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index f9162169..cbd9fe20 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -169,7 +169,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', 'ORDER BY bin' ].join('\n')); @@ -180,7 +180,8 @@ var TYPE = 'histogram'; type: 'histogram', options: { column: 'name', - bins: 10 // OPTIONAL + bins: 10, // OPTIONAL + aggregation: 'day' // OPTIONAL } } */ From d3bcf6f80d4d1d2802609a55116c796e4478fe9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 6 Jun 2017 09:15:49 +0200 Subject: [PATCH 163/402] Do not omit nonexistent property --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index cbd9fe20..1bf24df2 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -396,7 +396,7 @@ Histogram.prototype.format = function(result, override) { firstRow.min; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'bins_start'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); }); } From f14a61528ac4e5c0b13b7d0d2631fbc81114c969 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 6 Jun 2017 10:52:37 +0200 Subject: [PATCH 164/402] Upgrade to camshaft 0.55.3 --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 930001d8..f4ec8e44 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.9.1 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3). + ## 3.9.0 Released 2017-05-31 diff --git a/package.json b/package.json index 230ca8ff..3cd60f3b 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.2", + "camshaft": "cartodb/camshaft#0a3d24e936301e40b6bea1925b3c47f61cd24617", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", From 45d1d07ea27347f35e0e7340e0658a6f78edaee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 6 Jun 2017 17:21:33 +0200 Subject: [PATCH 165/402] Use released version of comshaft (0.55.3) --- package.json | 2 +- yarn.lock | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 3cd60f3b..b13e49c7 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "cartodb/camshaft#0a3d24e936301e40b6bea1925b3c47f61cd24617", + "camshaft": "0.55.3", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 089b43ad..f647ca8f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -150,7 +150,7 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.1.7: +brace-expansion@^1.0.0: version "1.1.7" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" dependencies: @@ -198,9 +198,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.2: - version "0.55.2" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.2.tgz#ee756633b7cde16891874c3441b48b5178ff55a2" +camshaft@0.55.3: + version "0.55.3" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.3.tgz#45fc7fa7b779b5ad19ad92cce16fca4e4cf5a3ec" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -209,7 +209,7 @@ camshaft@0.55.2: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -235,7 +235,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1745,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1770,7 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.81.0: +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1835,22 +1835,22 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@~4.3.3: - version "4.3.6" - resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" -semver@^5.1.0, semver@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - semver@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540" +semver@~4.3.3: + version "4.3.6" + resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" + semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" @@ -2107,7 +2107,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2115,7 +2115,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: From 02bf1dd2d7dee13071b815a8ba21466f22a81680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 6 Jun 2017 17:34:38 +0200 Subject: [PATCH 166/402] Release 3.9.1 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index f4ec8e44..8953bd0c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.9.1 -Released 2017-mm-dd +Released 2017-06-06 Announcements: - Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3). From 42e2f9e4b1f360af341db7739895c23bab4fbf83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 7 Jun 2017 15:54:19 +0200 Subject: [PATCH 167/402] Update commented use cases --- lib/cartodb/models/dataview/histogram.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 1bf24df2..43b6ee86 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -176,13 +176,23 @@ var dateHistogramQueryTpl = dot.template([ var TYPE = 'histogram'; /** - { - type: 'histogram', - options: { - column: 'name', - bins: 10, // OPTIONAL - aggregation: 'day' // OPTIONAL - } +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} + +Time series: +{ + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) + timezone: -7200 // OPTIONAL (UTC offset in seconds) + } } */ function Histogram(query, options, queries) { From 35d4fb4d27ca48a04643df41bea0f5f57fc14e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 7 Jun 2017 16:11:09 +0200 Subject: [PATCH 168/402] Improve readability, using extract method pattern --- lib/cartodb/models/dataview/histogram.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 43b6ee86..01827b54 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -399,11 +399,7 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; - binsStart = override.hasOwnProperty('start') ? - getBinStart(override) : - firstRow.hasOwnProperty('timestamp') ? - firstRow.timestamp : - firstRow.min; + binsStart = populateBinStart(override, firstRow); buckets = result.rows.map(function(row) { return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); @@ -457,6 +453,20 @@ function getTimezone(timezone) { return '' + timezoneInHours; } +function populateBinStart(override, firstRow) { + var binStart; + + if (override.hasOwnProperty('start')) { + binStart = getBinStart(override); + } else if (firstRow.hasOwnProperty('timestamp')) { + binStart = firstRow.timestamp; + } else { + binStart = firstRow.min; + } + + return binStart; +} + Histogram.prototype.getType = function() { return TYPE; }; From 6c301403e37e4f1901c5d43d54e5928e2de20d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 15:59:33 +0200 Subject: [PATCH 169/402] Histogram going red: fails while quering Infinity and NanN values --- test/acceptance/dataviews/histogram.js | 86 +++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 10 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index abbaef61..c75e9baf 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -3,6 +3,15 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + describe('histogram-dataview', function() { afterEach(function(done) { @@ -13,15 +22,6 @@ describe('histogram-dataview', function() { } }); - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - var mapConfig = createMapConfig( [ { @@ -89,7 +89,6 @@ describe('histogram-dataview', function() { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('pop_max_histogram', params, function(err, res) { assert.ok(!err, err); - assert.ok(res.errors); assert.equal(res.errors.length, 1); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); @@ -98,3 +97,70 @@ describe('histogram-dataview', function() { }); }); }); + + +describe('histogram-dataview: special float valuer', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_histogram', {}, function(err, dataview) { + assert.ok(!err, err); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); From cd53eda0a5062551fb7f64efaa11c82acd70bec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 16:01:41 +0200 Subject: [PATCH 170/402] Handle Infinity and NaN values for histograms --- lib/cartodb/models/dataview/histogram.js | 75 +++++++++++++++++++++--- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5d102bf5..f6b2635d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -13,12 +13,27 @@ var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join(' \n')); @@ -27,7 +42,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join('\n')); @@ -38,7 +53,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -57,7 +72,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -77,11 +92,34 @@ var nullsQueryTpl = dot.template([ ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + 'infinities AS (', + ' SELECT', + ' count(*) AS infinities_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + 'nans AS (', + ' SELECT', + ' count(*) AS nans_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', + ' infinities_count,', + ' nans_count,', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -91,9 +129,8 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'FROM filtered_source, basics, nulls, infinities, nans, bins', + 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', 'ORDER BY bin' ].join('\n')); @@ -168,7 +205,12 @@ Histogram.prototype.sql = function(psql, override, callback) { var _query = this.query; - var basicsQuery, binsQuery; + var filteredQuery, basicsQuery, binsQuery; + + filteredQuery = filteredQueryTpl({ + _query: _query, + _column: _column + }); if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { debug('overriding with %j', override); @@ -215,11 +257,20 @@ Histogram.prototype.sql = function(psql, override, callback) { var histogramSql = [ "WITH", [ + filteredQuery, basicsQuery, binsQuery, nullsQueryTpl({ _query: _query, _column: _column + }), + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ + _query: _query, + _column: _column }) ].join(',\n'), histogramQueryTpl({ @@ -241,6 +292,8 @@ Histogram.prototype.format = function(result, override) { var width = getWidth(override); var binsStart = getBinStart(override); var nulls = 0; + var infinities = 0; + var nans = 0; var avg; if (result.rows.length) { @@ -249,10 +302,12 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; + infinities = firstRow.infinities_count; + nans = firstRow.nans_count; binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'infinities_count', 'nans_count', 'avg_val'); }); } @@ -261,6 +316,8 @@ Histogram.prototype.format = function(result, override) { bins_count: binsCount, bins_start: binsStart, nulls: nulls, + infinities: infinities, + nans: nans, avg: avg, bins: buckets }; From 2772fc62d2971b9ab23bea3a0c66acba08a27ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 18:38:44 +0200 Subject: [PATCH 171/402] Use a set/dict for checking the existence --- lib/cartodb/backends/dataview.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 8e991b30..55fe81a0 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,7 +20,15 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } -var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'quarter', 'year']; +var DATE_AGGREGATIONS = { + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; module.exports = DataviewBackend; @@ -108,7 +116,7 @@ function getOverrideParams(params, ownFilter) { {ownFilter: ownFilter} ); - if (params.aggregation && DATE_AGGREGATIONS.indexOf(params.aggregation) !== -1) { + if (params.aggregation && DATE_AGGREGATIONS.hasOwnProperty(params.aggregation)) { overrideParams.aggregation = params.aggregation; } From 29a6658e3dbf622e5d4ea8a2e20fbc4ce08d533b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 19:22:33 +0200 Subject: [PATCH 172/402] Migrate dataviews endpoints to use the allow-query-params --- lib/cartodb/controllers/base.js | 14 +------ lib/cartodb/controllers/layergroup.js | 56 +++++++++++++++++++++------ 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 91fcaba7..39ed62c3 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -17,18 +17,8 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'zoom', 'lon', 'lat', - // widgets & filters - 'filters', // json - 'own_filter', // 0, 1 - 'bbox', // w,s,e,n - 'bins', // number - 'start', // number - 'end', // number - 'column_type', // string - 'aggregation', //string - 'timezone', // number - // widgets search - 'q' + // analysis + 'filters' // json ]; function BaseController(authApi, pgConnection) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index e0c7a7a3..fcab20d2 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -79,19 +79,51 @@ LayergroupController.prototype.register = function(app) { // Undocumented/non-supported API endpoint methods. // Use at your own peril. - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); + var allowedDataviewQueryParams = [ + 'filters', // json + 'own_filter', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'timezone', // number + 'q' // widgets search + ]; + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); app.get(app.base_url_mapconfig + '/:token/analysis/node/:nodeId', cors(), userMiddleware, From fe5c76d65b5aed1d21645b3f0b268cb24680c400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 19:25:05 +0200 Subject: [PATCH 173/402] Remove jshint hook --- lib/cartodb/backends/dataview.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 55fe81a0..e54b3a83 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -40,7 +40,6 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param mapConfigProvider.getMapConfig(this); }, function runDataviewQuery(err, mapConfig) { - /* jshint maxcomplexity: 7 */ assert.ifError(err); var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); From 7a3498e8ec2553629f89b0d2298386905e4b2bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 9 Jun 2017 12:17:16 +0200 Subject: [PATCH 174/402] Going red: formula does not work with Infinity or NaN values --- test/acceptance/dataviews/formula.js | 80 ++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/acceptance/dataviews/formula.js diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js new file mode 100644 index 00000000..93db6b38 --- /dev/null +++ b/test/acceptance/dataviews/formula.js @@ -0,0 +1,80 @@ +require('../../support/test_helper'); +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + +describe('formula-dataview: special float valuer', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_formula: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: 'val', + operation: 'avg' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_formula', {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.result, 501) + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); From 9ba65bd5a482e08a9141148f270ef7636cd8580d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 9 Jun 2017 15:18:37 +0200 Subject: [PATCH 175/402] Going green: Fix test for formula overviews flavour --- lib/cartodb/models/dataview/formula.js | 24 ++++++++++++++--- test/acceptance/dataviews/overviews.js | 36 +++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 156985c5..1421032d 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -7,9 +7,21 @@ dot.templateSettings.strip = false; var formulaQueryTpl = dot.template([ 'SELECT', - '{{=it._operation}}({{=it._column}}) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + ' {{=it._operation}}({{=it._column}}) AS result,', + ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + ' {{?it._operation!==\'count\'}}', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + ' {{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._operation!==\'count\'}}', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float', + '{{?}}' ].join('\n')); var VALID_OPERATIONS = { @@ -78,13 +90,17 @@ Formula.prototype.format = function(result) { var formattedResult = { operation: this.operation, result: 0, - nulls: 0 + nulls: 0, + nans: 0, + infinities: 0 }; if (result.rows.length) { formattedResult.operation = this.operation; formattedResult.result = result.rows[0].result; formattedResult.nulls = result.rows[0].nulls_count; + formattedResult.nans = result.rows[0].nans_count; + formattedResult.infinities = result.rows[0].infinities_count; } return formattedResult; diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index d23519ae..18fc1240 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -248,7 +248,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 5, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -260,7 +267,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -407,7 +421,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -419,7 +440,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); From 0aae29fb4b64e8dddb7baa3c136fda8db9ea830a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 9 Jun 2017 15:28:18 +0200 Subject: [PATCH 176/402] Fix jshint typo --- lib/cartodb/models/dataview/formula.js | 6 ++++-- test/acceptance/dataviews/formula.js | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 1421032d..04b89835 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -10,8 +10,10 @@ var formulaQueryTpl = dot.template([ ' {{=it._operation}}({{=it._column}}) AS result,', ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', ' {{?it._operation!==\'count\'}}', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', ' {{?}}', 'FROM ({{=it._query}}) _cdb_formula', '{{?it._operation!==\'count\'}}', diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js index 93db6b38..585a3b82 100644 --- a/test/acceptance/dataviews/formula.js +++ b/test/acceptance/dataviews/formula.js @@ -71,7 +71,7 @@ describe('formula-dataview: special float valuer', function() { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('val_formula', {}, function(err, dataview) { assert.ok(!err, err); - assert.equal(dataview.result, 501) + assert.equal(dataview.result, 501); assert.ok(dataview.infinities === (250 + 250)); assert.ok(dataview.nans === 250); done(); From e6aededf089fff0bb326ad291453d4585f400163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 17:19:05 +0200 Subject: [PATCH 177/402] Fix typo --- test/acceptance/dataviews/formula.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js index 585a3b82..941dc07e 100644 --- a/test/acceptance/dataviews/formula.js +++ b/test/acceptance/dataviews/formula.js @@ -11,7 +11,7 @@ function createMapConfig(layers, dataviews, analysis) { }; } -describe('formula-dataview: special float valuer', function() { +describe('formula-dataview: special float values', function() { afterEach(function(done) { if (this.testClient) { From ba6dc62a38245f1e18c47600ba165fb312d55e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 18:15:39 +0200 Subject: [PATCH 178/402] Going red: implementet test to check special float values support --- test/acceptance/dataviews/aggregation.js | 82 ++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 259cd2af..988f71ff 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -146,3 +146,85 @@ describe('aggregations happy cases', function() { }); }); }); + +describe.only('aggregation-dataview: special float values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_aggregation: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_aggregation', {}, function(err, dataview) { + assert.ifError(err); + assert.equal(dataview.result, 501); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); From e60bb770db7d0e6fe836a927d16db23df58a04d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 18:55:33 +0200 Subject: [PATCH 179/402] Summarizes infinity and NaN values --- lib/cartodb/models/dataview/aggregation.js | 83 +++++++++++++++------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index c15f0506..8c6fff54 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -5,11 +5,28 @@ var debug = require('debug')('windshaft:widget:aggregation'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' AND', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' count(1) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float THEN 1 ELSE 0 END) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -57,9 +74,9 @@ var rankedAggregationQueryTpl = dot.template([ var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'ORDER BY value DESC' ].join('\n')); @@ -128,6 +145,8 @@ Aggregation.prototype.sql = function(psql, override, callback) { var aggregationSql; + + if (!!override.ownFilter) { aggregationSql = [ this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), @@ -157,28 +176,34 @@ Aggregation.prototype.sql = function(psql, override, callback) { Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { return [ - "WITH", - [ - summaryQueryTpl({ - _query: query, - _column: column - }), - rankedCategoriesQueryTpl({ - _query: query, - _column: column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: query, - _column: column - }), - categoriesSummaryCountQueryTpl({ - _query: query, - _column: column - }) - ].join(',\n') - ].join('\n'); + "WITH", + [ + filteredQueryTpl({ + _query: this.query, + _column: this.column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + summaryQueryTpl({ + _query: query, + _column: column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + rankedCategoriesQueryTpl({ + _query: query, + _column: column, + _aggregation: this.getAggregationSql(), + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + categoriesSummaryMinMaxQueryTpl({ + _query: query, + _column: column + }), + categoriesSummaryCountQueryTpl({ + _query: query, + _column: column + }) + ].join(',\n') + ].join('\n'); }; var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); @@ -193,6 +218,8 @@ Aggregation.prototype.format = function(result) { var categories = []; var count = 0; var nulls = 0; + var nans = 0; + var infinities = 0; var minValue = 0; var maxValue = 0; var categoriesCount = 0; @@ -202,12 +229,14 @@ Aggregation.prototype.format = function(result) { var firstRow = result.rows[0]; count = firstRow.count; nulls = firstRow.nulls_count; + nans = firstRow.nans_count; + infinities = firstRow.infinities_count; minValue = firstRow.min_val; maxValue = firstRow.max_val; categoriesCount = firstRow.categories_count; result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count', 'nans_count', 'infinities_count')); }); } @@ -215,6 +244,8 @@ Aggregation.prototype.format = function(result) { aggregation: this.aggregation, count: count, nulls: nulls, + nans: nans, + infinities: infinities, min: minValue, max: maxValue, categoriesCount: categoriesCount, From 7b5111614c89e1b61f8dbb14cd701edb9ec98c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:21:41 +0200 Subject: [PATCH 180/402] Summarize special float values for ranked aggregation --- lib/cartodb/models/dataview/aggregation.js | 8 ++++---- test/acceptance/dataviews/aggregation.js | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 8c6fff54..145b6e9d 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -61,15 +61,15 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', - ' count, categories_count', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' ].join('\n')); var aggregationQueryTpl = dot.template([ @@ -145,7 +145,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { var aggregationSql; - + console.log('override.ownFilter', override.ownFilter); if (!!override.ownFilter) { aggregationSql = [ diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 988f71ff..d4f3d65c 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -217,14 +217,16 @@ describe.only('aggregation-dataview: special float values', function() { ] ); - it('should filter infinities out and count them in the summary', function(done) { - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('val_aggregation', {}, function(err, dataview) { - assert.ifError(err); - assert.equal(dataview.result, 501); - assert.ok(dataview.infinities === (250 + 250)); - assert.ok(dataview.nans === 250); - done(); + [{ own_filter: 0 }, {}].forEach(function (filter) { + it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.equal(dataview.result, 501); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); }); - }); + }) }); From 75d07745e6496d4bef923381196f22c0d1758572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:22:37 +0200 Subject: [PATCH 181/402] Improve readability --- test/acceptance/dataviews/aggregation.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index d4f3d65c..02b19782 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -217,7 +217,8 @@ describe.only('aggregation-dataview: special float values', function() { ] ); - [{ own_filter: 0 }, {}].forEach(function (filter) { + var filters = [{ own_filter: 0 }, {}]; + filters.forEach(function (filter) { it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { From 962fa055747f25548b23479b4c746cc2c25ae63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:28:49 +0200 Subject: [PATCH 182/402] Remove console --- lib/cartodb/models/dataview/aggregation.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 145b6e9d..f06f6091 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -145,8 +145,6 @@ Aggregation.prototype.sql = function(psql, override, callback) { var aggregationSql; - console.log('override.ownFilter', override.ownFilter); - if (!!override.ownFilter) { aggregationSql = [ this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), From 8b2fa27ba75f5f488cc7f6ffc7501f66a747d761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:45:06 +0200 Subject: [PATCH 183/402] Calculate aggregation filtering out special float values --- lib/cartodb/models/dataview/aggregation.js | 2 +- test/acceptance/dataviews/aggregation.js | 23 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index f06f6091..9ce889da 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -35,7 +35,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 02b19782..71aec370 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -187,7 +187,7 @@ describe.only('aggregation-dataview: special float values', function() { type: 'aggregation', options: { column: 'cat', - aggregation: 'sum', + aggregation: 'avg', aggregationColumn: 'val' } } @@ -217,15 +217,34 @@ describe.only('aggregation-dataview: special float values', function() { ] ); + // the_geom_webmercator | val | cat + // ----------------------+-----------+------------ + // | -Infinity | category_2 + // | NaN | category_1 + // | 3 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 7 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 11 | category_2 + // | " | " + var filters = [{ own_filter: 0 }, {}]; filters.forEach(function (filter) { it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { assert.ifError(err); - assert.equal(dataview.result, 501); assert.ok(dataview.infinities === (250 + 250)); assert.ok(dataview.nans === 250); + assert.ok(dataview.categories.length === 1); + dataview.categories.forEach(function (category) { + assert.ok(category.category === 'category_2'); + assert.ok(category.value === 501); + }) done(); }); }); From cb7ec5d556e4020b697fd82b71e270cc095a8988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:49:58 +0200 Subject: [PATCH 184/402] Fix jshint typos --- lib/cartodb/models/dataview/aggregation.js | 18 +++++++++++++----- test/acceptance/dataviews/aggregation.js | 4 ++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 9ce889da..6e46ea31 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -25,7 +25,13 @@ var summaryQueryTpl = dot.template([ ' SELECT', ' count(1) AS count,', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float THEN 1 ELSE 0 END) AS infinities_count,', + ' sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' @@ -61,12 +67,13 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', + ' min_val, max_val, count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' @@ -234,7 +241,8 @@ Aggregation.prototype.format = function(result) { categoriesCount = firstRow.categories_count; result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count', 'nans_count', 'infinities_count')); + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', + 'max_val', 'categories_count', 'nans_count', 'infinities_count')); }); } diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 71aec370..4f7c0062 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -244,9 +244,9 @@ describe.only('aggregation-dataview: special float values', function() { dataview.categories.forEach(function (category) { assert.ok(category.category === 'category_2'); assert.ok(category.value === 501); - }) + }); done(); }); }); - }) + }); }); From 227937bf4c933488728587ad8e034f4ce248a20c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:50:24 +0200 Subject: [PATCH 185/402] Remove test filter --- test/acceptance/dataviews/aggregation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 4f7c0062..5d9dc499 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -147,7 +147,7 @@ describe('aggregations happy cases', function() { }); }); -describe.only('aggregation-dataview: special float values', function() { +describe('aggregation-dataview: special float values', function() { afterEach(function(done) { if (this.testClient) { From 3ae66e41437160dfd3cf9bb2cac52cb0df195d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 09:30:43 +0200 Subject: [PATCH 186/402] Do not filter special values out if aggregation column is not defined --- lib/cartodb/models/dataview/aggregation.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 6e46ea31..71574ac5 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -11,12 +11,14 @@ var filteredQueryTpl = dot.template([ ' FROM ({{=it._query}}) _cdb_filtered_source', ' WHERE', ' {{=it._column}} IS NOT NULL', + ' {{?it._aggregationColumn}}', ' AND', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'NaN\'::float', + ' {{?}}', ')' ].join(' \n')); From 551b6d409a1820fd0a2acc534ff02c246c51e777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 09:44:53 +0200 Subject: [PATCH 187/402] Remove bad condition --- lib/cartodb/models/dataview/aggregation.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 71574ac5..e61b1454 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -9,10 +9,8 @@ var filteredQueryTpl = dot.template([ 'filtered_source AS (', ' SELECT *', ' FROM ({{=it._query}}) _cdb_filtered_source', - ' WHERE', - ' {{=it._column}} IS NOT NULL', ' {{?it._aggregationColumn}}', - ' AND', + ' WHERE', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', From c08db78a0bc496ea290180eed210091af1f4eba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 19:01:28 +0200 Subject: [PATCH 188/402] Going red: implement test to check aggregation with overviews support special float values --- test/acceptance/dataviews/overviews.js | 58 +++++++++++++++++++++ test/support/sql/windshaft.test.sql | 72 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 18fc1240..c822caee 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() { params: { query: 'select * from test_table_overviews' } + }, + { + id: 'data-source-special-float-values', + type: 'source', + params: { + query: 'select * from test_special_float_values_table_overviews' + } } ], dataviews: { @@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() { aggregationColumn: 'name', } }, + test_categories_special_values: { + type: 'aggregation', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'name', + aggregation: 'sum', + aggregationColumn: 'value', + } + }, test_histogram: { type: 'histogram', source: {id: 'data-source'}, @@ -202,6 +220,17 @@ describe('dataviews using tables with overviews', function() { cartocss_version: '2.3.0', source: { id: 'data-source' } } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_special_float_values_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { + id: 'data-source-special-float-values' + } + } } ] }; @@ -473,5 +502,34 @@ describe('dataviews using tables with overviews', function() { }); + describe('aggregation special float values', function () { + var params = {}; + + it("should expose an aggregation dataview", function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_categories_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + aggregation: 'sum', + count: 5, + nulls: 0, + nans: 1, + infinities: 1, + min: 6, + max: 6, + categoriesCount: 3, + categories:[ + { category: 'El Rey del Tallarín', value: null, agg: false }, + { category: 'El Lacón', value: null, agg: false }, + { category: 'Hawai', value: 6, agg: false } + ], + type: 'aggregation' + }); + testClient.drain(done); + }); + }); + }); }); }); diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 91484950..996c6b3e 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES INSERT INTO _vovw_1_test_table_overviews VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5); +-- table with overviews whit special float values + +CREATE TABLE test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id; + +SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true); + +ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass); + +INSERT INTO test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1), +(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1), +(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1); + +ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id); + +CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom); +CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE TABLE _vovw_1_test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER; + +INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2); -- analysis tables ----------------------------------------------- From 668b22628ce4ce5cfbf589ef50ab4868e549c11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 19:01:59 +0200 Subject: [PATCH 189/402] Going green: support special float values in aggergation dataview with overviews --- .../models/dataview/overviews/aggregation.js | 63 +++++++++++++++---- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index da63b27f..1b0e2ec6 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,14 +1,38 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../aggregation'); +var debug = require('debug')('windshaft:widget:aggregation:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' {{?it._aggregationColumn}}', + ' WHERE', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float', + ' {{?}}', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' sum(_feature_count) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', + ' sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -17,7 +41,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', @@ -43,21 +67,23 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'ORDER BY value DESC' ].join('\n')); @@ -84,21 +110,28 @@ Aggregation.prototype.sql = function(psql, override, callback) { } var _query = this.rewrittenQuery(this.query); + var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ "WITH", [ + filteredQueryTpl({ + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn + }), summaryQueryTpl({ _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -120,15 +153,21 @@ Aggregation.prototype.sql = function(psql, override, callback) { aggregationSql = [ "WITH", [ + filteredQueryTpl({ + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn + }), summaryQueryTpl({ _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -147,6 +186,8 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } + debug(aggregationSql); + return callback(null, aggregationSql); }; From 3f17c8b15ab9074b1004d08aee23f59656fba42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 15:05:46 +0200 Subject: [PATCH 190/402] Filter out special float values before categorizing them --- lib/cartodb/models/dataview/overviews/aggregation.js | 4 ++-- test/acceptance/dataviews/overviews.js | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 1b0e2ec6..cc95e73c 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -60,7 +60,7 @@ var categoriesSummaryCountQueryTpl = dot.template([ ' SELECT count(1) AS categories_count', ' FROM (', ' SELECT {{=it._column}} AS category', - ' FROM ({{=it._query}}) _cdb_categories', + ' FROM filtered_source', ' GROUP BY {{=it._column}}', ' ) _cdb_categories_count', ')' @@ -82,7 +82,7 @@ var rankedAggregationQueryTpl = dot.template([ var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', - 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', + 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count', 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'ORDER BY value DESC' ].join('\n')); diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index c822caee..08ee6b7c 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -505,7 +505,7 @@ describe('dataviews using tables with overviews', function() { describe('aggregation special float values', function () { var params = {}; - it("should expose an aggregation dataview", function (done) { + it("should expose an aggregation dataview filtering special float values out", function (done) { var testClient = new TestClient(overviewsMapConfig); testClient.getDataview('test_categories_special_values', params, function (err, dataview) { if (err) { @@ -519,12 +519,8 @@ describe('dataviews using tables with overviews', function() { infinities: 1, min: 6, max: 6, - categoriesCount: 3, - categories:[ - { category: 'El Rey del Tallarín', value: null, agg: false }, - { category: 'El Lacón', value: null, agg: false }, - { category: 'Hawai', value: 6, agg: false } - ], + categoriesCount: 1, + categories: [ { category: 'Hawai', value: 6, agg: false } ], type: 'aggregation' }); testClient.drain(done); From 06d40e8b1e0288e3a6d477c90f40c7fa03869274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 18:57:26 +0200 Subject: [PATCH 191/402] Going red: add test to check that special float values are not being filtered out when the layer uses overviews --- test/acceptance/dataviews/overviews.js | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 08ee6b7c..20a7ca9d 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -178,6 +178,16 @@ describe('dataviews using tables with overviews', function() { bins: 2 } }, + test_histogram_special_values: { + type: 'histogram', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + bins: 2 + } + }, test_avg: { type: 'formula', source: {id: 'data-source'}, @@ -526,6 +536,27 @@ describe('dataviews using tables with overviews', function() { testClient.drain(done); }); }); + + it('should expose a histogram dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_histogram_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + bin_width: 0, + bins_count: 1, + bins_start: 3, + nulls: 0, + infinities: 1, + nans: 1, + avg: 3, + bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ], + type: 'histogram' + }); + testClient.drain(done); + }); + }); }); }); }); From dee00e6abd10e78583c3d0871e26f73bda989e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:00:37 +0200 Subject: [PATCH 192/402] Going green: handle special float values for histogram when overviews are involved --- .../models/dataview/overviews/histogram.js | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 67da4514..ed88a621 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -12,12 +12,27 @@ var columnTypeQueryTpl = dot.template( var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join(' \n')); @@ -26,7 +41,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join('\n')); @@ -37,7 +52,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -56,7 +71,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -76,11 +91,34 @@ var nullsQueryTpl = dot.template([ ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + 'infinities AS (', + ' SELECT', + ' count(*) AS infinities_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + 'nans AS (', + ' SELECT', + ' count(*) AS nans_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', + ' infinities_count,', + ' nans_count,', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -90,9 +128,8 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', ' sum(_feature_count) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'FROM filtered_source, basics, nulls, infinities, nans, bins', + 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', 'ORDER BY bin' ].join('\n')); @@ -156,7 +193,12 @@ Histogram.prototype.sql = function(psql, override, callback) { var _query = this.rewrittenQuery(this.query); - var basicsQuery, binsQuery; + var filteredQuery, basicsQuery, binsQuery; + + filteredQuery = filteredQueryTpl({ + _query: _query, + _column: _column + }); if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { basicsQuery = overrideBasicsQueryTpl({ @@ -202,11 +244,20 @@ Histogram.prototype.sql = function(psql, override, callback) { var histogramSql = [ "WITH", [ + filteredQuery, basicsQuery, binsQuery, nullsQueryTpl({ _query: _query, _column: _column + }), + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ + _query: _query, + _column: _column }) ].join(',\n'), histogramQueryTpl({ From ef849aec342a588ee359f9af70e83480a2a90d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:19:08 +0200 Subject: [PATCH 193/402] Going red: add test to check that special float values are not being filtered out in formula dataview when the layer uses overviews --- test/acceptance/dataviews/overviews.js | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 20a7ca9d..a024febd 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -196,6 +196,16 @@ describe('dataviews using tables with overviews', function() { operation: 'avg' } }, + test_sum_special_values: { + type: 'formula', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + operation: 'sum' + } + }, test_count: { type: 'formula', source: {id: 'data-source'}, @@ -557,6 +567,24 @@ describe('dataviews using tables with overviews', function() { testClient.drain(done); }); }); + + it('should expose a formula (sum) dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_sum_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + operation: 'sum', + result: 6, + nulls: 0, + nans: 1, + infinities: 1, + type: 'formula' + }); + testClient.drain(done); + }); + }); }); }); }); From 81f60959e59901e8f412d9f13a83f17aee62a230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:20:39 +0200 Subject: [PATCH 194/402] Going green: handle special float values for formula when overviews are involved --- .../models/dataview/overviews/formula.js | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 9e331f0b..ba98b2c8 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -14,14 +14,34 @@ var formulaQueryTpls = { 'sum': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), 'avg': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), }; From 7d0af4e2596982fb154d2ad4bf495f11d9d254fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:20:39 +0200 Subject: [PATCH 195/402] Going green: handle special float values for formula when overviews are involved --- .../models/dataview/overviews/formula.js | 28 ++++++-- test/acceptance/dataviews/overviews.js | 65 ++++++++++++++++--- 2 files changed, 80 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 9e331f0b..ba98b2c8 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -14,14 +14,34 @@ var formulaQueryTpls = { 'sum': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), 'avg': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), }; diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index a024febd..4362d7a3 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -196,7 +196,7 @@ describe('dataviews using tables with overviews', function() { operation: 'avg' } }, - test_sum_special_values: { + test_formula_sum_special_values: { type: 'formula', source: { id: 'data-source-special-float-values' @@ -261,7 +261,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -273,7 +280,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":3, + "nulls":0, + "type":"formula", + "infinities": 0, + "nans": 0 + }); testClient.drain(done); }); @@ -285,7 +299,12 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":5, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -338,7 +357,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -435,7 +461,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -446,7 +479,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -514,7 +554,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -570,7 +617,7 @@ describe('dataviews using tables with overviews', function() { it('should expose a formula (sum) dataview filtering special float values out', function (done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_sum_special_values', params, function (err, dataview) { + testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) { if (err) { return done(err); } From 443c1100d7487b2f2d29a8a023eb3bf0d7675f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 16:31:24 +0200 Subject: [PATCH 196/402] Formula dataview: support special values only if column is a float column --- lib/cartodb/models/dataview/formula.js | 45 +++++++++--- .../models/dataview/overviews/formula.js | 68 +++++++++++++++---- test/acceptance/dataviews/overviews.js | 13 +++- 3 files changed, 102 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 04b89835..1d3204e7 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -5,25 +5,25 @@ var debug = require('debug')('windshaft:widget:formula'); var dot = require('dot'); dot.templateSettings.strip = false; +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); + var formulaQueryTpl = dot.template([ 'SELECT', ' {{=it._operation}}({{=it._column}}) AS result,', ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - ' {{?it._operation!==\'count\'}}', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', - ' {{?}}', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._operation!==\'count\'}}', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float', - '{{?}}' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')); var VALID_OPERATIONS = { @@ -34,6 +34,11 @@ var VALID_OPERATIONS = { max: true }; +var FLOAT_OIDS = { + 700: true, + 701: true +}; + var TYPE = 'formula'; /** @@ -63,6 +68,7 @@ function Formula(query, options) { this.query = query; this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = new BaseWidget(); @@ -71,13 +77,36 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } var _query = this.query; + + var columnTypeQuery = columnTypeQueryTpl({ + column: this.column, query: _query + }); + + if (this._isFloatColumn === null) { + var readOnlyTransaction = true; + psql.query(columnTypeQuery, function(err, result) { + self._isFloatColumn = false; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (FLOAT_OIDS.hasOwnProperty(pgType)) { + self._isFloatColumn = true; + } + } + self.sql(psql, override, callback); + }, readOnlyTransaction); + return null; + } + var formulaSql = formulaQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _operation: this.operation, _column: this.column diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index ba98b2c8..a1437c73 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,47 +1,61 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); +var debug = require('debug')('windshaft:widget:formula:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var FLOAT_OIDS = { + 700: true, + 701: true +}; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); + var formulaQueryTpls = { 'count': dot.template([ 'SELECT', 'sum(_feature_count) AS result,', '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula' ].join('\n')), 'sum': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')), 'avg': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')), }; @@ -49,6 +63,7 @@ function Formula(query, options, queryRewriter, queryRewriteData, params) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -57,20 +72,45 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; var formulaQueryTpl = formulaQueryTpls[this.operation]; if ( formulaQueryTpl ) { // supported formula for use with overviews + + var columnTypeQuery = columnTypeQueryTpl({ + column: this.column, query: this.query + }); + + if (this._isFloatColumn === null) { + var readOnlyTransaction = true; + psql.query(columnTypeQuery, function(err, result) { + self._isFloatColumn = false; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (FLOAT_OIDS.hasOwnProperty(pgType)) { + self._isFloatColumn = true; + } + } + self.sql(psql, override, callback); + }, readOnlyTransaction); + return null; + } + var formulaSql = formulaQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: this.rewrittenQuery(this.query), _operation: this.operation, _column: this.column }); callback = callback || override; + debug(formulaSql); + return callback(null, formulaSql); } + // default behaviour return this.defaultSql(psql, override, callback); }; diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 4362d7a3..dcff687f 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -303,7 +303,9 @@ describe('dataviews using tables with overviews', function() { "operation":"count", "result":5, "nulls":0, - "type":"formula" + "type":"formula", + "infinities": 0, + "nans": 0 }); testClient.drain(done); @@ -498,7 +500,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":1, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); From ad570ab6f23d8b5b5e017dcecf75b09e4af7ee0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 18:04:35 +0200 Subject: [PATCH 197/402] Use dataview base to get column type in formula dataviews --- lib/cartodb/models/dataview/base.js | 34 +++++ lib/cartodb/models/dataview/formula.js | 31 +---- .../models/dataview/overviews/formula.js | 121 ++++++++---------- 3 files changed, 92 insertions(+), 94 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index b2e2f188..d7479b6d 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -1,3 +1,6 @@ +var dot = require('dot'); +dot.templateSettings.strip = false; + function BaseDataview() {} module.exports = BaseDataview; @@ -24,3 +27,34 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { BaseDataview.prototype.search = function(psql, userQuery, callback) { return callback(null, this.format({ rows: [] })); }; + +var FLOAT_OIDS = { + 700: true, + 701: true +}; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1' +); + +BaseDataview.prototype.getColumnType = function (psql, column, query, callback) { + var readOnlyTransaction = true; + + var columnTypeQuery = columnTypeQueryTpl({ + column: column, query: query + }); + + psql.query(columnTypeQuery, function(err, result) { + if (err) { + return callback(err); + } + var pgType = result.rows[0].pg_typeof; + callback(null, getPGTypeName(pgType)); + }, readOnlyTransaction); +}; + +function getPGTypeName (pgType) { + return { + float: FLOAT_OIDS.hasOwnProperty(pgType) + }; +} diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 1d3204e7..2272089c 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -5,10 +5,6 @@ var debug = require('debug')('windshaft:widget:formula'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var formulaQueryTpl = dot.template([ 'SELECT', ' {{=it._operation}}({{=it._column}}) AS result,', @@ -34,11 +30,6 @@ var VALID_OPERATIONS = { max: true }; -var FLOAT_OIDS = { - 700: true, - 701: true -}; - var TYPE = 'formula'; /** @@ -84,30 +75,20 @@ Formula.prototype.sql = function(psql, override, callback) { override = {}; } - var _query = this.query; - - var columnTypeQuery = columnTypeQueryTpl({ - column: this.column, query: _query - }); - if (this._isFloatColumn === null) { - var readOnlyTransaction = true; - psql.query(columnTypeQuery, function(err, result) { - self._isFloatColumn = false; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (FLOAT_OIDS.hasOwnProperty(pgType)) { - self._isFloatColumn = true; - } + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.query, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; } self.sql(psql, override, callback); - }, readOnlyTransaction); + }); return null; } var formulaSql = formulaQueryTpl({ _isFloatColumn: this._isFloatColumn, - _query: _query, + _query: this.query, _operation: this.operation, _column: this.column }); diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index a1437c73..533dd921 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -5,58 +5,49 @@ var debug = require('debug')('windshaft:widget:formula:overview'); var dot = require('dot'); dot.templateSettings.strip = false; -var FLOAT_OIDS = { - 700: true, - 701: true -}; - -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var formulaQueryTpls = { - 'count': dot.template([ - 'SELECT', - 'sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), - 'sum': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' - ].join('\n')), - 'avg': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' - ].join('\n')), + 'count': dot.template([ + 'SELECT', + 'sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'sum': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), + 'avg': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), }; function Formula(query, options, queryRewriter, queryRewriteData, params) { @@ -71,38 +62,30 @@ Formula.prototype.constructor = Formula; module.exports = Formula; -Formula.prototype.sql = function(psql, override, callback) { +Formula.prototype.sql = function (psql, override, callback) { var self = this; var formulaQueryTpl = formulaQueryTpls[this.operation]; - if ( formulaQueryTpl ) { + if (formulaQueryTpl) { // supported formula for use with overviews - - var columnTypeQuery = columnTypeQueryTpl({ - column: this.column, query: this.query - }); - if (this._isFloatColumn === null) { - var readOnlyTransaction = true; - psql.query(columnTypeQuery, function(err, result) { - self._isFloatColumn = false; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (FLOAT_OIDS.hasOwnProperty(pgType)) { - self._isFloatColumn = true; - } + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.query, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; } self.sql(psql, override, callback); - }, readOnlyTransaction); + }); return null; } var formulaSql = formulaQueryTpl({ _isFloatColumn: this._isFloatColumn, - _query: this.rewrittenQuery(this.query), + _query: this.rewrittenQuery(this.query), _operation: this.operation, - _column: this.column + _column: this.column }); + callback = callback || override; debug(formulaSql); From 849caf9b585c317b09a8a3b8f9c8c0f1b10c4b94 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 15 Jun 2017 18:30:13 +0200 Subject: [PATCH 198/402] Upgrade to cammshaft 0.55.4 (#702) --- NEWS.md | 6 ++++++ package.json | 4 ++-- yarn.lock | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8953bd0c..b8208751 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.9.2 +Released 2017-06-16 + +Announcements: + - Upgrades camshaft to [0.55.4](https://github.com/CartoDB/camshaft/releases/tag/0.55.4). + ## 3.9.1 Released 2017-06-06 diff --git a/package.json b/package.json index b13e49c7..8258d70e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.1", + "version": "3.9.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.3", + "camshaft": "0.55.4", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index f647ca8f..038778ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -198,9 +198,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.3: - version "0.55.3" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.3.tgz#45fc7fa7b779b5ad19ad92cce16fca4e4cf5a3ec" +camshaft@0.55.4: + version "0.55.4" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.4.tgz#9b83e2fd4adc0f471976d7b0ef319e28b8864adc" dependencies: async "^1.5.2" bunyan "1.8.1" From 2a366ec16fb6eecf805e7000643d27c85f38536f Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 15 Jun 2017 18:32:28 +0200 Subject: [PATCH 199/402] Stubs next version --- NEWS.md | 5 +++++ package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b8208751..b9a32405 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 3.9.3 +Released 2017-mm-dd + + + ## 3.9.2 Released 2017-06-16 diff --git a/package.json b/package.json index 8258d70e..0176e5ff 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.2", + "version": "3.9.3", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 115d8fe6854f6971690a2a8ad7d55b02ed802b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 19:07:31 +0200 Subject: [PATCH 200/402] Handle special float values only when aggregation columns is float --- lib/cartodb/models/dataview/aggregation.js | 63 ++++++++++++++++------ 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index e61b1454..0d576614 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -9,14 +9,12 @@ var filteredQueryTpl = dot.template([ 'filtered_source AS (', ' SELECT *', ' FROM ({{=it._query}}) _cdb_filtered_source', - ' {{?it._aggregationColumn}}', - ' WHERE', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', ' AND', - ' {{=it._aggregationColumn}} != \'NaN\'::float', - ' {{?}}', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -24,15 +22,15 @@ var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' count(1) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', - ' sum(', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', ' CASE', ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', ' THEN 1', ' ELSE 0', ' END', ' ) AS infinities_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -68,22 +66,24 @@ var categoriesSummaryCountQueryTpl = dot.template([ var rankedAggregationQueryTpl = dot.template([ 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', - ' min_val, max_val, count, categories_count, nans_count, infinities_count', + ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); @@ -108,7 +108,7 @@ var TYPE = 'aggregation'; } } */ -function Aggregation(query, options) { +function Aggregation(query, options, queries) { if (!_.isString(options.column)) { throw new Error('Aggregation expects `column` in widget options'); } @@ -132,9 +132,11 @@ function Aggregation(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = new BaseWidget(); @@ -143,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var _query = this.query; var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -164,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } else { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', @@ -179,16 +208,18 @@ Aggregation.prototype.sql = function(psql, override, callback) { return callback(null, aggregationSql); }; -Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { +Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { return [ "WITH", [ filteredQueryTpl({ + _isFloatColumn: isFloatColumn, _query: this.query, _column: this.column, _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null }), summaryQueryTpl({ + _isFloatColumn: isFloatColumn, _query: query, _column: column, _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null From 20d7f1a7c56749d8e18acb930c58b558c7b1bc12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 19:22:26 +0200 Subject: [PATCH 201/402] Handle special float values only when aggregation columns is float (overviews) --- .../models/dataview/overviews/aggregation.js | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index cc95e73c..730ba822 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -9,14 +9,12 @@ var filteredQueryTpl = dot.template([ 'filtered_source AS (', ' SELECT *', ' FROM ({{=it._query}}) _cdb_filtered_source', - ' {{?it._aggregationColumn}}', - ' WHERE', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', ' AND', - ' {{=it._aggregationColumn}} != \'NaN\'::float', - ' {{?}}', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -24,15 +22,15 @@ var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' sum(_feature_count) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', - ' sum(', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', ' CASE', ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', ' THEN 1', ' ELSE 0', ' END', ' ) AS infinities_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -68,34 +66,38 @@ var categoriesSummaryCountQueryTpl = dot.template([ var rankedAggregationQueryTpl = dot.template([ 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); var CATEGORIES_LIMIT = 6; - function Aggregation(query, options, queryRewriter, queryRewriteData, params) { + function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -104,6 +106,8 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; @@ -112,17 +116,30 @@ Aggregation.prototype.sql = function(psql, override, callback) { var _query = this.rewrittenQuery(this.query); var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ "WITH", [ filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn }), summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn @@ -143,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -154,11 +172,13 @@ Aggregation.prototype.sql = function(psql, override, callback) { "WITH", [ filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn }), summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn @@ -179,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _limit: CATEGORIES_LIMIT From 8e95cf20c06c3ff0f9cf0af57d57718f81a6e3d5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 16 Jun 2017 11:13:40 +0200 Subject: [PATCH 202/402] Upgrade camshaft to 0.55.5 (#703) --- NEWS.md | 5 +++-- package.json | 4 ++-- yarn.lock | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/NEWS.md b/NEWS.md index b9a32405..87ec5032 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,10 @@ # Changelog ## 3.9.3 -Released 2017-mm-dd - +Released 2017-06-16 +Announcements: + - Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5). ## 3.9.2 Released 2017-06-16 diff --git a/package.json b/package.json index 0176e5ff..d034e5c6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.3", + "version": "3.9.4", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.4", + "camshaft": "0.55.5", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 038778ce..3bae7d4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -198,9 +198,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.4: - version "0.55.4" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.4.tgz#9b83e2fd4adc0f471976d7b0ef319e28b8864adc" +camshaft@0.55.5: + version "0.55.5" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.5.tgz#6831f74022b06e12ddab8e00953c7cc859598ac4" dependencies: async "^1.5.2" bunyan "1.8.1" From e2b976d9d0b8f039cdacf5da9acc655edde9ad4a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 16 Jun 2017 11:16:17 +0200 Subject: [PATCH 203/402] Fix version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d034e5c6..c32459f0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.4", + "version": "3.9.3", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From b1ac5b8ca9475dddb859a2e539e0538d062c0f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 16 Jun 2017 12:57:46 +0200 Subject: [PATCH 204/402] Handle special float values only if column is float --- lib/cartodb/models/dataview/base.js | 9 +- lib/cartodb/models/dataview/histogram.js | 101 +++++++++-------- .../models/dataview/overviews/histogram.js | 104 ++++++++++-------- 3 files changed, 120 insertions(+), 94 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index d7479b6d..2cd24261 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -33,6 +33,12 @@ var FLOAT_OIDS = { 701: true }; +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + var columnTypeQueryTpl = dot.template( 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1' ); @@ -55,6 +61,7 @@ BaseDataview.prototype.getColumnType = function (psql, column, query, callback) function getPGTypeName (pgType) { return { - float: FLOAT_OIDS.hasOwnProperty(pgType) + float: FLOAT_OIDS.hasOwnProperty(pgType), + date: DATE_OIDS.hasOwnProperty(pgType) }; } diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index f6b2635d..73cc8724 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -5,9 +5,6 @@ var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; @@ -19,12 +16,12 @@ var filteredQueryTpl = dot.template([ ' FROM ({{=it._query}}) _cdb_filtered_source', ' WHERE', ' {{=it._column}} IS NOT NULL', - ' AND', + ' {{?it._isFloatColumn}}AND', ' {{=it._column}} != \'infinity\'::float', ' AND', ' {{=it._column}} != \'-infinity\'::float', ' AND', - ' {{=it._column}} != \'NaN\'::float', + ' {{=it._column}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -118,8 +115,8 @@ var histogramQueryTpl = dot.template([ ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', - ' infinities_count,', - ' nans_count,', + ' {{?it._isFloatColumn}}infinities_count,', + ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -129,8 +126,9 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, basics, nulls, infinities, nans, bins', - 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', + 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'GROUP BY bin, bins_number, bin_width, nulls_count,', + ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -164,55 +162,49 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.queries.no_filters - }); - if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction return null; } + var histogramSql = this._buildQuery(override); + + return callback(null, histogramSql); +}; + +Histogram.prototype._buildQuery = function (override) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; + var _query = this.query; + if (this._columnType === 'date') { _column = columnCastTpl({column: _column}); } - var _query = this.query; - - var filteredQuery, basicsQuery, binsQuery; - filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }); - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, @@ -232,7 +224,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -253,17 +245,18 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }), + if (this._columnType === 'float') { + cteSql.push( infinitiesQueryTpl({ _query: _query, _column: _column @@ -272,8 +265,14 @@ Histogram.prototype.sql = function(psql, override, callback) { _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) @@ -281,7 +280,15 @@ Histogram.prototype.sql = function(psql, override, callback) { debug(histogramSql); - return callback(null, histogramSql); + return histogramSql; +}; + +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); }; Histogram.prototype.format = function(result, override) { diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index ed88a621..361cc8f2 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -1,14 +1,11 @@ var _ = require('underscore'); var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../histogram'); +var debug = require('debug')('windshaft:dataview:histogram:overview'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -18,12 +15,12 @@ var filteredQueryTpl = dot.template([ ' FROM ({{=it._query}}) _cdb_filtered_source', ' WHERE', ' {{=it._column}} IS NOT NULL', - ' AND', + ' {{?it._isFloatColumn}}AND', ' {{=it._column}} != \'infinity\'::float', ' AND', ' {{=it._column}} != \'-infinity\'::float', ' AND', - ' {{=it._column}} != \'NaN\'::float', + ' {{=it._column}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -117,8 +114,8 @@ var histogramQueryTpl = dot.template([ ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', - ' infinities_count,', - ' nans_count,', + ' {{?it._isFloatColumn}}infinities_count,', + ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -128,8 +125,9 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', ' sum(_feature_count) AS freq', - 'FROM filtered_source, basics, nulls, infinities, nans, bins', - 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', + 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + ' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -149,36 +147,23 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; - -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.rewrittenQuery(this.queries.no_filters) - }); if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction @@ -191,16 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) { return this.defaultSql(psql, override, callback); } + var histogramSql = this._buildQuery(override); + + return callback(null, histogramSql); +}; + +Histogram.prototype._buildQuery = function (override) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; var _query = this.rewrittenQuery(this.query); - var filteredQuery, basicsQuery, binsQuery; - filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }); - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { + debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, _column: _column, @@ -219,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -240,17 +233,18 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }), + if (this._columnType === 'float') { + cteSql.push( infinitiesQueryTpl({ _query: _query, _column: _column @@ -259,12 +253,30 @@ Histogram.prototype.sql = function(psql, override, callback) { _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) ].join('\n'); - return callback(null, histogramSql); + debug(histogramSql); + + return histogramSql; }; + +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); +}; + + From 6bfedef7eb75ed78f470ea1a003b7262ce45afdb Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Mon, 19 Jun 2017 12:47:08 +0200 Subject: [PATCH 205/402] Cast histogram width bucket to timestamp --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 01827b54..b7bb29a4 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -155,7 +155,7 @@ var dateHistogramQueryTpl = dot.template([ ' CASE WHEN min_val = max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', - ' WIDTH_BUCKET({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\', bins_array),', + ' WIDTH_BUCKET({{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\', bins_array),', ' bins_number', ' )) - 1', ' END AS bin,', From e44d418db39dec200ffcdfa8ca6d03f0d717a40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Jun 2017 18:44:21 +0200 Subject: [PATCH 206/402] Going red: Add test to check that aggregation dataview supports numeric special value --- test/acceptance/dataviews/aggregation.js | 46 ++++++++++++++++++++++++ test/support/test-client.js | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 5d9dc499..86eb0613 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -190,6 +190,17 @@ describe('aggregation-dataview: special float values', function() { aggregation: 'avg', aggregationColumn: 'val' } + }, + sum_aggregation_numeric: { + source: { + id: 'a1' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } } }, [ @@ -213,10 +224,31 @@ describe('aggregation-dataview: special float values', function() { 'FROM generate_series(1, 1000) x' ].join('\n') } + }, { + "id": "a1", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 3 = 0 THEN \'NaN\'::numeric', + ' WHEN x % 3 = 1 THEN x', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } } ] ); + // Source a0 + // ----------------------------------------------- // the_geom_webmercator | val | cat // ----------------------+-----------+------------ // | -Infinity | category_2 @@ -248,5 +280,19 @@ describe('aggregation-dataview: special float values', function() { done(); }); }); + + it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.ok(dataview.nans === 333); + assert.ok(dataview.categories.length === 2); + dataview.categories.forEach(function (category) { + assert.ok(category.value !== null); + }); + done(); + }); + }); + }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 04980a5a..659d6686 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } params.placeholders = params.placeholders || {}; - + assert.response(server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), From 5555b8ad8ec76025cb1770336271e1802b526477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Jun 2017 18:59:36 +0200 Subject: [PATCH 207/402] Going green: support numeric NaN values for dataviews --- lib/cartodb/models/dataview/base.js | 3 ++- test/acceptance/dataviews/aggregation.js | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 2cd24261..f8eb1027 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -30,7 +30,8 @@ BaseDataview.prototype.search = function(psql, userQuery, callback) { var FLOAT_OIDS = { 700: true, - 701: true + 701: true, + 1700: true }; var DATE_OIDS = { diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 86eb0613..697004e8 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -293,6 +293,5 @@ describe('aggregation-dataview: special float values', function() { done(); }); }); - }); }); From 6f7cb75256e6d75a19979b160305a5b8a00e91d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Jun 2017 20:19:02 +0200 Subject: [PATCH 208/402] Fix bad datetime conversion --- lib/cartodb/models/dataview/histogram.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b7bb29a4..dbaa002f 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -155,7 +155,10 @@ var dateHistogramQueryTpl = dot.template([ ' CASE WHEN min_val = max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', - ' WIDTH_BUCKET({{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\', bins_array),', + ' WIDTH_BUCKET(', + ' {{=it._column}}::timestamp WITHOUT TIME ZONE AT TIME ZONE \'{{=it._timezone}}\',', + ' bins_array', + ' ),', ' bins_number', ' )) - 1', ' END AS bin,', From 14e71b929abf24674cfe654de12d6377c8173b15 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 22 Jun 2017 17:48:55 +0200 Subject: [PATCH 209/402] Release version 3.9.4 --- NEWS.md | 6 ++ package.json | 4 +- yarn.lock | 153 ++++++++++++++++++++++++++------------------------- 3 files changed, 85 insertions(+), 78 deletions(-) diff --git a/NEWS.md b/NEWS.md index 87ec5032..9f334355 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.9.4 +Released 2017-06-22 + +Announcements: + - Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6). + ## 3.9.3 Released 2017-06-16 diff --git a/package.json b/package.json index c32459f0..10d11d0d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.3", + "version": "3.9.4", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.5", + "camshaft": "0.55.6", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 3bae7d4a..e0ff12ef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -53,8 +53,8 @@ ap@~0.2.0: resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" aproba@^1.0.3: - version "1.1.1" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" + version "1.1.2" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" are-we-there-yet@~1.1.2: version "1.1.4" @@ -109,9 +109,9 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -balanced-match@^0.4.1: - version "0.4.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -150,21 +150,17 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.0.0: - version "1.1.7" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" +brace-expansion@^1.1.7: + version "1.1.8" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" dependencies: - balanced-match "^0.4.1" + balanced-match "^1.0.0" concat-map "0.0.1" browser-stdout@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" -buffer-shims@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" - buffer-writer@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" @@ -198,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.5: - version "0.55.5" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.5.tgz#6831f74022b06e12ddab8e00953c7cc859598ac4" +camshaft@0.55.6: + version "0.55.6" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.6.tgz#11af28051c3b911fb023ae1cafb165bbd040f174" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -209,7 +205,7 @@ camshaft@0.55.5: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -227,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -421,10 +417,10 @@ debug@2.6.0: ms "0.7.2" debug@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" + version "1.0.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" dependencies: - ms "0.6.2" + ms "2.0.0" decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" @@ -712,7 +708,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: +generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -724,6 +720,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" +generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.6" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -792,8 +792,8 @@ growl@1.9.2: resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: - version "4.0.8" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" + version "4.0.10" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -899,7 +899,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1: +inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -991,12 +991,6 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" -jodid25519@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" - dependencies: - jsbn "~0.1.0" - js-base64@^2.1.9: version "2.1.9" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" @@ -1253,10 +1247,10 @@ mime@~1.2.11: resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: - brace-expansion "^1.0.0" + brace-expansion "^1.1.7" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1277,8 +1271,8 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi minimist "0.0.8" mocha@~3.4.1: - version "3.4.1" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.1.tgz#a3802b4aa381934cacb38de70cf771621da8f9af" + version "3.4.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" dependencies: browser-stdout "1.3.0" commander "2.9.0" @@ -1296,10 +1290,6 @@ moment@^2.10.6: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" -ms@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.6.2.tgz#d89c2124c6fdc1353d65a8b77bf1aac4b193708c" - ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" @@ -1308,6 +1298,10 @@ ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1348,8 +1342,8 @@ nock@~2.11.0: propagate "0.3.x" node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: - version "0.6.34" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" + version "0.6.36" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1404,10 +1398,14 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1508,23 +1506,23 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.7.1" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" + version "1.8.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" dependencies: generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: - version "1.11.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.11.0.tgz#aae91a82d952b633bb88d006350a166daaf6ea90" + version "1.12.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" dependencies: ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" - postgres-interval "~1.0.0" + postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +"pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1537,8 +1535,8 @@ pg@cartodb/node-postgres#6.1.2-cdb1: semver "4.3.2" pgpass@1.x: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.1.tgz#0de8b5bef993295d90a7e17d976f568dcd25d49f" + version "1.0.2" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" dependencies: split "^1.0.0" @@ -1601,9 +1599,9 @@ postgres-date@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" -postgres-interval@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.0.2.tgz#7261438d862b412921c6fdb7617668424b73a6ed" +postgres-interval@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.0.tgz#1031e7bac34564132862adc9eb6c6d2f3aa75bb4" dependencies: xtend "^4.0.0" @@ -1708,14 +1706,14 @@ readable-stream@1.1, readable-stream@~1.1.9: string_decoder "~0.10.x" readable-stream@^2.0.6, readable-stream@^2.1.4: - version "2.2.9" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" + version "2.3.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.2.tgz#5a04df05e4f57fe3f0dc68fdd11dc5c97c7e6f4d" dependencies: - buffer-shims "~1.0.0" core-util-is "~1.0.0" - inherits "~2.0.1" + inherits "~2.0.3" isarray "~1.0.0" process-nextick-args "~1.0.6" + safe-buffer "~5.1.0" string_decoder "~1.0.0" util-deprecate "~1.0.1" @@ -1745,7 +1743,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1768,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1827,7 +1825,11 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1: +safe-buffer@^5.0.1, safe-buffer@~5.1.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" + +safe-buffer@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" @@ -1981,8 +1983,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" + version "1.13.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -1991,7 +1993,6 @@ sshpk@^1.7.0: optionalDependencies: bcrypt-pbkdf "^1.0.0" ecc-jsbn "~0.1.1" - jodid25519 "^1.0.0" jsbn "~0.1.0" tweetnacl "~0.14.0" @@ -2030,10 +2031,10 @@ string_decoder@~0.10.x: resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" string_decoder@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" + version "1.0.2" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.2.tgz#b29e1f4e1125fa97a10382b8a533737b7491e179" dependencies: - buffer-shims "~1.0.0" + safe-buffer "~5.0.1" stringstream@~0.0.4: version "0.0.5" @@ -2107,7 +2108,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2115,7 +2116,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2193,8 +2194,8 @@ type-is@~1.6.10, type-is@~1.6.6: mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.26" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.26.tgz#3a1db8ae0a0aba7f92e1ddadadbd0293d549f90e" + version "2.8.29" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -2234,8 +2235,8 @@ utils-merge@1.0.0: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" uuid@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" validate-npm-package-license@^3.0.1: version "3.0.1" From a6daca96283c8e304395d9068eb11a4bc4e0759a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 22 Jun 2017 18:04:23 +0200 Subject: [PATCH 210/402] Support date histograms using timestamp with and without timezones --- lib/cartodb/models/dataview/histogram.js | 12 +- test/acceptance/dataviews/histogram.js | 372 ++++++++++++----------- 2 files changed, 210 insertions(+), 174 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dbaa002f..3409e6fb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -137,9 +137,13 @@ var dateBinsQueryTpl = dot.template([ ' SELECT', ' ARRAY(', ' {{?it._aggregation==="quarter"}}', - ' SELECT GENERATE_SERIES(start_date, end_date, \'3 month\'::interval)', + ' SELECT GENERATE_SERIES(start_date::timestamptz, end_date::timestamptz, \'3 month\'::interval)', ' {{??}}', - ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' SELECT GENERATE_SERIES(', + ' start_date::timestamptz,', + ' end_date::timestamptz,', + ' \'1 {{=it._aggregation}}\'::interval', + ' )', ' {{?}}', ' ) AS bins_array', ' FROM basics', @@ -156,7 +160,7 @@ var dateHistogramQueryTpl = dot.template([ ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp WITHOUT TIME ZONE AT TIME ZONE \'{{=it._timezone}}\',', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\',', ' bins_array', ' ),', ' bins_number', @@ -164,7 +168,7 @@ var dateHistogramQueryTpl = dot.template([ ' END AS bin,', ' min(date_part(', ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', + ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}}::timestamptz', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 0efb56d6..f39db632 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -134,6 +134,17 @@ describe('histogram-dataview for date column type', function() { aggregation: 'month', timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } + }, + date_histogram_tz: { + source: { + id: 'date-histogram-source-tz' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } } }, [ @@ -148,205 +159,226 @@ describe('histogram-dataview for date column type', function() { ") date" ].join(' ') } + }, + { + "id": "date-histogram-source-tz", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", + ") date" + ].join(' ') + } } ] ); - it('should create a date histogram aggregated in months (EDT)', function (done) { - var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with timezone', + dataviewId: 'date_histogram_tz' + }, { + desc: 'supporting timestamp without timezone', + dataviewId: 'date_histogram' + }]; - this.testClient = new TestClient(mapConfig, 1234); + dateHistogramsUseCases.forEach(function (test) { + it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { + var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes - this.testClient.getDataview('date_histogram', {}, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); + this.testClient = new TestClient(mapConfig, 1234); - var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); + this.testClient.getDataview(test.dataviewId, {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_EDT_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should override aggregation in weeks', function (done) { - var params = { - aggregation: 'week' - }; + it('should override aggregation in weeks ' + test.desc, function (done) { + var params = { + aggregation: 'week' + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 61); - dataview.bins.forEach(function(bin) { - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should override start and end', function (done) { - var params = { - start: 1180659600, // 2007-06-01 01:00:00 - end: 1193792400 // 2007-10-31 01:00:00 - }; + it('should override start and end ' + test.desc, function (done) { + var params = { + start: 1180659600, // 2007-06-01 01:00:00 + end: 1193792400 // 2007-10-31 01:00:00 + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 6); - dataview.bins.forEach(function(bin) { - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should aggregate histogram overriding default timezone to CEST', function (done) { - var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) - var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) - var params = { - timezone: TIMEZONE_CEST_IN_SECONDS - }; + it('should aggregate histogram overriding default timezone to CEST ' + test.desc, function (done) { + var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + var params = { + timezone: TIMEZONE_CEST_IN_SECONDS + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); - var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_CEST_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); + }); - done(); + it('should aggregate histogram overriding default timezone to UTC/GMT ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); }); }); - - it('should aggregate histogram overriding default timezone to UTC/GMT', function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC - var params = { - timezone: TIMEZONE_UTC_IN_SECONDS - }; - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); - - var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); - }); - - done(); - }); - }); - - it('should aggregate histogram using "quarter" aggregation', function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC - var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, - aggregation: 'quarter' - }; - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 6); - - var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .add(index * 3, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); - }); - - done(); - }); - }); - }); From 2dae09c35b43b251bffc3c641c312010fee08c71 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 22 Jun 2017 18:11:58 +0200 Subject: [PATCH 211/402] Fix yarn.lock and update HOWTO_RELEASE In previous commit I updated all the dependencies with the newest versions when I really intended to update just one. After asking the node gurus we agreed on changing the docs to reflect a less aggressive strategy for dependency upgrades. --- HOWTO_RELEASE | 2 +- yarn.lock | 147 +++++++++++++++++++++++++------------------------- 2 files changed, 74 insertions(+), 75 deletions(-) diff --git a/HOWTO_RELEASE b/HOWTO_RELEASE index be552a95..a2927b60 100644 --- a/HOWTO_RELEASE +++ b/HOWTO_RELEASE @@ -1,7 +1,7 @@ 1. Test (make clean all check), fix if broken before proceeding 2. Ensure proper version in package.json 3. Ensure NEWS section exists for the new version, review it, add release date -4. Recreate yarn.lock with: `yarn upgrade` +4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}` 5. Commit package.json, yarn.lock, NEWS 6. git tag -a Major.Minor.Patch # use NEWS section as content 7. Stub NEWS/package for next version diff --git a/yarn.lock b/yarn.lock index e0ff12ef..8409e6e1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -53,8 +53,8 @@ ap@~0.2.0: resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" aproba@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" + version "1.1.1" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" are-we-there-yet@~1.1.2: version "1.1.4" @@ -109,9 +109,9 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" +balanced-match@^0.4.1: + version "0.4.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -150,17 +150,21 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.1.7: - version "1.1.8" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" +brace-expansion@^1.0.0: + version "1.1.7" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" dependencies: - balanced-match "^1.0.0" + balanced-match "^0.4.1" concat-map "0.0.1" browser-stdout@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" +buffer-shims@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" + buffer-writer@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" @@ -205,7 +209,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +227,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -417,10 +421,10 @@ debug@2.6.0: ms "0.7.2" debug@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" + version "1.0.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" dependencies: - ms "2.0.0" + ms "0.6.2" decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" @@ -708,7 +712,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3: +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -720,10 +724,6 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -792,8 +792,8 @@ growl@1.9.2: resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: - version "4.0.10" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" + version "4.0.8" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -899,7 +899,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@~2.0.0, inherits@~2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -991,6 +991,12 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" +jodid25519@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" + dependencies: + jsbn "~0.1.0" + js-base64@^2.1.9: version "2.1.9" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" @@ -1247,10 +1253,10 @@ mime@~1.2.11: resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + version "3.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" dependencies: - brace-expansion "^1.1.7" + brace-expansion "^1.0.0" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1271,8 +1277,8 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi minimist "0.0.8" mocha@~3.4.1: - version "3.4.2" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" + version "3.4.1" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.1.tgz#a3802b4aa381934cacb38de70cf771621da8f9af" dependencies: browser-stdout "1.3.0" commander "2.9.0" @@ -1290,6 +1296,10 @@ moment@^2.10.6: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" +ms@0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.6.2.tgz#d89c2124c6fdc1353d65a8b77bf1aac4b193708c" + ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" @@ -1298,10 +1308,6 @@ ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" -ms@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1342,8 +1348,8 @@ nock@~2.11.0: propagate "0.3.x" node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: - version "0.6.36" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" + version "0.6.34" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1398,14 +1404,10 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0: +object-assign@4.1.0, object-assign@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1506,23 +1508,23 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.8.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" + version "1.7.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" dependencies: generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: - version "1.12.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" + version "1.11.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.11.0.tgz#aae91a82d952b633bb88d006350a166daaf6ea90" dependencies: ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" - postgres-interval "^1.1.0" + postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1535,8 +1537,8 @@ pg-types@1.*: semver "4.3.2" pgpass@1.x: - version "1.0.2" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" + version "1.0.1" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.1.tgz#0de8b5bef993295d90a7e17d976f568dcd25d49f" dependencies: split "^1.0.0" @@ -1599,9 +1601,9 @@ postgres-date@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" -postgres-interval@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.0.tgz#1031e7bac34564132862adc9eb6c6d2f3aa75bb4" +postgres-interval@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.0.2.tgz#7261438d862b412921c6fdb7617668424b73a6ed" dependencies: xtend "^4.0.0" @@ -1706,14 +1708,14 @@ readable-stream@1.1, readable-stream@~1.1.9: string_decoder "~0.10.x" readable-stream@^2.0.6, readable-stream@^2.1.4: - version "2.3.2" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.2.tgz#5a04df05e4f57fe3f0dc68fdd11dc5c97c7e6f4d" + version "2.2.9" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" dependencies: + buffer-shims "~1.0.0" core-util-is "~1.0.0" - inherits "~2.0.3" + inherits "~2.0.1" isarray "~1.0.0" process-nextick-args "~1.0.6" - safe-buffer "~5.1.0" string_decoder "~1.0.0" util-deprecate "~1.0.1" @@ -1743,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1768,7 +1770,7 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.81.0: +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1825,11 +1827,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@~5.1.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" - -safe-buffer@~5.0.1: +safe-buffer@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" @@ -1983,8 +1981,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.13.1" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" + version "1.13.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -1993,6 +1991,7 @@ sshpk@^1.7.0: optionalDependencies: bcrypt-pbkdf "^1.0.0" ecc-jsbn "~0.1.1" + jodid25519 "^1.0.0" jsbn "~0.1.0" tweetnacl "~0.14.0" @@ -2031,10 +2030,10 @@ string_decoder@~0.10.x: resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" string_decoder@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.2.tgz#b29e1f4e1125fa97a10382b8a533737b7491e179" + version "1.0.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" dependencies: - safe-buffer "~5.0.1" + buffer-shims "~1.0.0" stringstream@~0.0.4: version "0.0.5" @@ -2108,7 +2107,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2116,7 +2115,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2194,8 +2193,8 @@ type-is@~1.6.10, type-is@~1.6.6: mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.29" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" + version "2.8.26" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.26.tgz#3a1db8ae0a0aba7f92e1ddadadbd0293d549f90e" dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -2235,8 +2234,8 @@ utils-merge@1.0.0: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" uuid@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" + version "3.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" validate-npm-package-license@^3.0.1: version "3.0.1" From 15b88c6a67f023e831927dd81d5535d206aff532 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 22 Jun 2017 18:26:33 +0200 Subject: [PATCH 212/402] Stub next version --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 9f334355..5965ae8f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 3.9.5 +Released 2017-mm-dd + ## 3.9.4 Released 2017-06-22 diff --git a/package.json b/package.json index 10d11d0d..404524c1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.4", + "version": "3.9.5", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 32274e66fdb7aa3668af59ec2de2d7eb23d3bbf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 23 Jun 2017 12:24:22 +0200 Subject: [PATCH 213/402] Dataview formula: count infinities and nans as we do with nulls --- lib/cartodb/models/dataview/formula.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 2272089c..50869762 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -14,7 +14,7 @@ var formulaQueryTpl = dot.template([ ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', + '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', From 166e29e8ce9d95aac099df5282ff750d6314e077 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 23 Jun 2017 16:53:16 +0200 Subject: [PATCH 214/402] Forward queries parameter from overview dataviews to base dataviews --- lib/cartodb/models/dataview/overviews/aggregation.js | 2 +- lib/cartodb/models/dataview/overviews/base.js | 5 +++-- lib/cartodb/models/dataview/overviews/formula.js | 4 ++-- lib/cartodb/models/dataview/overviews/histogram.js | 2 +- lib/cartodb/models/dataview/overviews/list.js | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 730ba822..5df092f4 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -90,7 +90,7 @@ var aggregationQueryTpl = dot.template([ var CATEGORIES_LIMIT = 6; function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; this.queries = queries; diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index 1425e2d1..38b2c119 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -1,14 +1,15 @@ var _ = require('underscore'); var BaseDataview = require('../base'); -function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { +function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) { this.BaseDataview = BaseDataview; this.query = query; this.queryOptions = queryOptions; this.queryRewriter = queryRewriter; this.queryRewriteData = queryRewriteData; this.options = options; - this.baseDataview = new this.BaseDataview(this.query, this.queryOptions); + this.queries = queries; + this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries); } module.exports = BaseOverviewsDataview; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 533dd921..a30bc783 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -50,8 +50,8 @@ var formulaQueryTpls = { ].join('\n')), }; -function Formula(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function Formula(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.column = options.column || '1'; this.operation = options.operation; this._isFloatColumn = null; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 361cc8f2..6674f6a0 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -132,7 +132,7 @@ var histogramQueryTpl = dot.template([ ].join('\n')); function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; this.queries = queries; diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js index 7e3b3161..6ec731f4 100644 --- a/lib/cartodb/models/dataview/overviews/list.js +++ b/lib/cartodb/models/dataview/overviews/list.js @@ -1,8 +1,8 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../list'); -function List(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function List(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); } List.prototype = Object.create(BaseOverviewsDataview.prototype); From b2f3735e95d6c7820d7e1a18b494c1a997711221 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 23 Jun 2017 18:59:51 +0200 Subject: [PATCH 215/402] The formula widget wasn't using the no_filters query for checking column types --- lib/cartodb/models/dataview/formula.js | 5 +++-- lib/cartodb/models/dataview/overviews/formula.js | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 50869762..7ec356b7 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -41,7 +41,7 @@ var TYPE = 'formula'; } } */ -function Formula(query, options) { +function Formula(query, options, queries) { if (!_.isString(options.operation)) { throw new Error('Formula expects `operation` in widget options'); } @@ -57,6 +57,7 @@ function Formula(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column || '1'; this.operation = options.operation; this._isFloatColumn = null; @@ -77,7 +78,7 @@ Formula.prototype.sql = function(psql, override, callback) { if (this._isFloatColumn === null) { this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.query, function (err, type) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { if (!err && !!type) { self._isFloatColumn = type.float; } diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index a30bc783..64d612c9 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -55,6 +55,7 @@ function Formula(query, options, queryRewriter, queryRewriteData, params, querie this.column = options.column || '1'; this.operation = options.operation; this._isFloatColumn = null; + this.queries = queries; } Formula.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -70,7 +71,7 @@ Formula.prototype.sql = function (psql, override, callback) { // supported formula for use with overviews if (this._isFloatColumn === null) { this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.query, function (err, type) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { if (!err && !!type) { self._isFloatColumn = type.float; } From 13764e18ce3686f2b31bfefc9b32ed49242452fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 11:21:05 +0200 Subject: [PATCH 216/402] Going red: attributes service do not support special numeric values (Infinity, -Infinity, NaN) --- test/acceptance/special-numeric-values.js | 71 +++++++++++++++ test/support/test-client.js | 101 +++++++++++++++++++++- 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 test/acceptance/special-numeric-values.js diff --git a/test/acceptance/special-numeric-values.js b/test/acceptance/special-numeric-values.js new file mode 100644 index 00000000..2b596b8e --- /dev/null +++ b/test/acceptance/special-numeric-values.js @@ -0,0 +1,71 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); + +describe('special numeric values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var ATTRIBUTES_LAYER = 1; + + function createMapConfig(sql, id, columns) { + return { + version: '1.6.0', + layers: [ + { + type: 'mapnik', + options: { + sql: "select 1 as id, 'SRID=4326;POINT(0 0)'::geometry as the_geom", + cartocss: '#style { }', + cartocss_version: '2.0.1' + } + }, + { + type: 'mapnik', + options: { + sql: sql || "select 1 as i, 6 as n, 'SRID=4326;POINT(0 0)'::geometry as the_geom", + attributes: { + id: id || 'i', + columns: columns || ['n'] + }, + cartocss: '#style { }', + cartocss_version: '2.0.1' + } + } + ] + }; + } + + it('should retrieve special numeric values', function (done) { + var featureId = 1; + var sql = [ + 'SELECT', + ' 1 as cartodb_id,', + ' null::geometry the_geom_webmercator,', + ' \'infinity\'::float as infinity,', + ' \'-infinity\'::float as _infinity,', + ' \'NaN\'::float as nan' + ].join('\n'); + var id = 'cartodb_id'; + var columns = ['infinity', '_infinity', 'nan']; + + var mapConfig = createMapConfig(sql, id, columns); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getFeatureAttributes(featureId, ATTRIBUTES_LAYER, {}, function (err, attributes) { + assert.ifError(err); + assert.equal(attributes.infinity, 'Infinity'); + assert.equal(attributes._infinity, '-Infinity'); + assert.equal(attributes.nan, 'NaN'); + done(); + }); + }); +}); + diff --git a/test/support/test-client.js b/test/support/test-client.js index 04980a5a..8e9dcf0d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -406,6 +406,105 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }; +TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, callback) { + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + var extraParams = {}; + if (this.apiKey) { + extraParams.api_key = this.apiKey; + } + if (params && params.filters) { + extraParams.filters = JSON.stringify(params.filters); + } + + var url = '/api/v1/map'; + if (Object.keys(extraParams).length > 0) { + url += '?' + qs.stringify(extraParams); + } + + var expectedResponse = params.response || { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + var layergroupId; + step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + + var parsedBody = JSON.parse(res.body); + + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return next(null, parsedBody.layergroupid); + } + ); + }, + function getFeatureAttributes(err, layergroupId) { + assert.ifError(err); + + var next = this; + + url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId; + + assert.response(server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }, + expectedResponse, + function(res, err) { + if (err) { + return next(err); + } + + next(null, JSON.parse(res.body)); + } + ); + }, + function finish(err, attributes) { + if (err) { + return callback(err); + } + + return callback(null, attributes); + } + ); +}; + TestClient.prototype.getTile = function(z, x, y, params, callback) { var self = this; @@ -435,7 +534,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } params.placeholders = params.placeholders || {}; - + assert.response(server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), From 3e12bfe27aa8c96387b2da426189526ef178b574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 11:53:22 +0200 Subject: [PATCH 217/402] Going green: support special numeric values for json responses --- lib/cartodb/server.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 13d64804..deb88e16 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -310,6 +310,25 @@ function bootstrap(opts) { app.enable('jsonp callback'); app.disable('x-powered-by'); app.disable('etag'); + + // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 + // See: http://expressjs.com/en/4x/api.html#app.set + app.set('json replacer', function (key, value) { + if (value !== value) { + return 'NaN'; + } + + if (value === Infinity) { + return 'Infinity'; + } + + if (value === -Infinity) { + return '-Infinity'; + } + + return value; + }); + app.use(bodyParser.json()); app.use(function bootstrap$prepareRequestResponse(req, res, next) { From af42fba53bb0f964475a6883f49816de648808ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 14:28:23 +0200 Subject: [PATCH 218/402] Check that quarter aggreagtion uses filters properly in date histogram dataview --- test/acceptance/dataviews/histogram.js | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index f39db632..b26f0728 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -380,5 +380,49 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('bins_count should be equal to bins array length filtered by start and end ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870399 // 2008-06-30T23:59:59Z, last bin end + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 6); + assert.equal(dataview.bins_count, dataview.bins.length); + done(); + }); + }); + + it('bins_count should be greater than bins array length filtered by start and end ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 7); + assert.ok(dataview.bins_count > dataview.bins.length); + done(); + }); + }); }); }); From 01027b73da21d9fca87d054c90050599a86d649e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 14:36:18 +0200 Subject: [PATCH 219/402] Fix jshint typo --- test/acceptance/dataviews/histogram.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index b26f0728..93b59b6e 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -381,9 +381,8 @@ describe('histogram-dataview for date column type', function() { }); }); - it('bins_count should be equal to bins array length filtered by start and end ' + test.desc, function (done) { + it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) { var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC var params = { timezone: TIMEZONE_UTC_IN_SECONDS, aggregation: 'quarter', @@ -403,9 +402,8 @@ describe('histogram-dataview for date column type', function() { }); }); - it('bins_count should be greater than bins array length filtered by start and end ' + test.desc, function (done) { + it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) { var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC var params = { timezone: TIMEZONE_UTC_IN_SECONDS, aggregation: 'quarter', From 3a442bea44b3a03a5ed384f594a7585c27b2a4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 17:06:24 +0200 Subject: [PATCH 220/402] Update NEWS --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 5965ae8f..b96a7b94 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.9.5 Released 2017-mm-dd + - Dataviews: support special numeric values (±Infinity, NaN) #700 + + ## 3.9.4 Released 2017-06-22 From 2f1cacdfc795f390d05bf4441097185f3abaf1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 17:07:52 +0200 Subject: [PATCH 221/402] Release 3.9.5 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b96a7b94..64c33d4d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.9.5 -Released 2017-mm-dd +Released 2017-06-27 - Dataviews: support special numeric values (±Infinity, NaN) #700 From f507f7a74b2e9b6c3b5a52d60bc56b57b567d7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 17:18:08 +0200 Subject: [PATCH 222/402] Stub next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 64c33d4d..454b9142 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.6 +Released 2017-mm-dd + + ## 3.9.5 Released 2017-06-27 diff --git a/package.json b/package.json index 404524c1..463848c5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.5", + "version": "3.9.6", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 7018af18b6d1d5623a919bcf0e30ad86a6425de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Jun 2017 19:58:45 +0200 Subject: [PATCH 223/402] Support automatic aggregation for time-series histogram --- lib/cartodb/models/dataview/histogram.js | 99 +++++++++++++++++++++++- test/acceptance/dataviews/histogram.js | 22 ++++++ 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 3409e6fb..2070bab3 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -10,6 +10,47 @@ var columnTypeQueryTpl = dot.template( ); var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var dateIntervalQueryTpl = dot.template([ + 'WITH', + 'dates AS (', + ' SELECT', + ' MAX({{=it.column}}) AS _end,', + ' MIN({{=it.column}}) AS _start', + ' FROM ({{=it.query}}) _cdb_source', + '),', + 'interval_in_days AS (', + ' SELECT' , + ' DATE_PART(\'day\', _end - _start) AS days', + ' FROM dates', + '),', + 'interval_in_hours AS (', + ' SELECT', + ' days * 24 + DATE_PART(\'hour\', _end - _start) AS hours', + ' FROM interval_in_days, dates', + '),', + 'interval_in_minutes AS (', + ' SELECT', + ' hours * 60 + DATE_PART(\'minute\', _end - _start) AS minutes', + ' FROM interval_in_hours, dates', + '),', + 'interval_in_seconds AS (', + ' SELECT', + ' minutes * 60 + DATE_PART(\'second\', _end - _start) AS seconds', + ' FROM interval_in_minutes, dates', + ')', + 'SELECT', + ' ROUND(days / 365) AS year,', + ' ROUND(days / 90) AS quarter,', + ' ROUND(days / 30) AS month,', + ' ROUND(days / 7) AS week,', + ' days AS day,', + ' hours AS hour,', + ' minutes AS minute,', + ' seconds AS second', + 'FROM interval_in_days, interval_in_hours, interval_in_minutes, interval_in_seconds' +].join('\n')); + +var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -258,8 +299,8 @@ Histogram.prototype.sql = function(psql, override, callback) { return null; } - if (this._columnType === 'date' && this.aggregation !== undefined) { - return this._buildDateHistogramQuery(override, callback); + if (this._columnType === 'date') { + return this._buildDateHistogramQuery(psql, override, callback); } if (this._columnType === 'date') { @@ -331,12 +372,24 @@ Histogram.prototype.sql = function(psql, override, callback) { return callback(null, histogramSql); }; -Histogram.prototype._buildDateHistogramQuery = function (override, callback) { +Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var _timezone = override && Number.isFinite(override.timezone) ? override.timezone : this.timezone; + if (!_aggregation) { + this.getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + this._buildDateHistogramQuery(psql, override, callback); + }.bind(this)); + return null; + } + var dateBasicsQuery; if (override && _.has(override, 'start') && _.has(override, 'end')) { @@ -390,10 +443,49 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { return callback(null, histogramSql); }; +Histogram.prototype.getAutomaticAggregation = function (psql, callback) { + var dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + var aggegations = result.rows[0]; + var aggregation = Object.keys(aggegations) + .map(function (key) { + return { + name: key, + value: aggegations[key] + }; + }) + .reduce(function (closer, current) { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + var closerDiff = MAX_INTERVAL_VALUE - closer.value; + var currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; + var aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); @@ -414,6 +506,7 @@ Histogram.prototype.format = function(result, override) { } return { + aggregation: aggregation, bin_width: width, bins_count: binsCount, bins_start: binsStart, diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 93b59b6e..857aaa07 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -145,6 +145,15 @@ describe('histogram-dataview for date column type', function() { aggregation: 'month', timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } + }, + date_histogram_automatic: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd' + } } }, [ @@ -423,4 +432,17 @@ describe('histogram-dataview for date column type', function() { }); }); }); + + it('automatic mode', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); }); From 6d2934b30baf4f43cb63b62f2e100d140170c5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 29 Jun 2017 16:53:52 +0200 Subject: [PATCH 224/402] Going red: add test to check automatic mode works with dates --- lib/cartodb/models/dataview/histogram.js | 2 + test/acceptance/dataviews/histogram.js | 78 +++++++++++++++++++++--- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 2070bab3..b8b9085a 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -449,6 +449,8 @@ Histogram.prototype.getAutomaticAggregation = function (psql, callback) { column: this.column }); + debug(dateIntervalQuery); + psql.query(dateIntervalQuery, function (err, result) { if (err) { return callback(err); diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 857aaa07..c5927d52 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -116,7 +116,7 @@ describe('histogram-dataview for date column type', function() { "type": "cartodb", "options": { "source": { - "id": "date-histogram-source" + "id": "datetime-histogram-source" }, "cartocss": "#points { marker-width: 10; marker-fill: red; }", "cartocss_version": "2.3.0" @@ -124,9 +124,9 @@ describe('histogram-dataview for date column type', function() { } ], { - date_histogram: { + datetime_histogram: { source: { - id: 'date-histogram-source' + id: 'datetime-histogram-source' }, type: 'histogram', options: { @@ -135,9 +135,9 @@ describe('histogram-dataview for date column type', function() { timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, - date_histogram_tz: { + datetime_histogram_tz: { source: { - id: 'date-histogram-source-tz' + id: 'datetime-histogram-source-tz' }, type: 'histogram', options: { @@ -146,6 +146,25 @@ describe('histogram-dataview for date column type', function() { timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, + datetime_histogram_automatic: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd' + } + }, + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'year' + } + }, date_histogram_automatic: { source: { id: 'date-histogram-source' @@ -158,7 +177,7 @@ describe('histogram-dataview for date column type', function() { }, [ { - "id": "date-histogram-source", + "id": "datetime-histogram-source", "type": "source", "params": { "query": [ @@ -170,7 +189,7 @@ describe('histogram-dataview for date column type', function() { } }, { - "id": "date-histogram-source-tz", + "id": "datetime-histogram-source-tz", "type": "source", "params": { "query": [ @@ -180,16 +199,28 @@ describe('histogram-dataview for date column type', function() { ") date" ].join(' ') } + }, + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date::date AS d", + "from generate_series(", + "'2007-02-15'::date, '2008-04-09'::date, '1 day'::interval", + ") date" + ].join(' ') + } } ] ); var dateHistogramsUseCases = [{ desc: 'supporting timestamp with timezone', - dataviewId: 'date_histogram_tz' + dataviewId: 'datetime_histogram_tz' }, { desc: 'supporting timestamp without timezone', - dataviewId: 'date_histogram' + dataviewId: 'datetime_histogram' }]; dateHistogramsUseCases.forEach(function (test) { @@ -433,7 +464,34 @@ describe('histogram-dataview for date column type', function() { }); }); - it('automatic mode', function (done) { + it('should find the best aggregation (automatic mode) to build the histogram', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should work with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'year'); + assert.equal(dataview.bins.length, 2); + assert.equal(dataview.bins_count, 2); + done(); + }); + }); + + + it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) { var params = {}; this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) { From 468f641af8bf9896ed198f585d0d2b64c8f885ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 29 Jun 2017 16:57:27 +0200 Subject: [PATCH 225/402] Going green: automatic mode works with dates --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b8b9085a..4499dd6c 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -14,8 +14,8 @@ var dateIntervalQueryTpl = dot.template([ 'WITH', 'dates AS (', ' SELECT', - ' MAX({{=it.column}}) AS _end,', - ' MIN({{=it.column}}) AS _start', + ' MAX({{=it.column}}::timestamp) AS _end,', + ' MIN({{=it.column}}::timestamp) AS _start', ' FROM ({{=it.query}}) _cdb_source', '),', 'interval_in_days AS (', From 0684c1b9d34ffdee103bfee9f0cdb0697e04f7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 5 Jul 2017 19:08:19 +0200 Subject: [PATCH 226/402] Work in progress: get timeout from redis --- lib/cartodb/api/user_limits_api.js | 88 ++++++++++++++++--- .../provider/create-layergroup-provider.js | 2 +- .../mapconfig/provider/map-store-provider.js | 2 +- .../mapconfig/provider/named-map-provider.js | 2 +- test/acceptance/user-timeout-limit.js | 82 +++++++++++++++++ 5 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 test/acceptance/user-timeout-limit.js diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index ae9cd46b..7f244e7d 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -1,3 +1,5 @@ +var step = require('step'); + /** * * @param metadataBackend @@ -13,16 +15,82 @@ function UserLimitsApi(metadataBackend, options) { module.exports = UserLimitsApi; -UserLimitsApi.prototype.getRenderLimits = function (username, callback) { +UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) { var self = this; - this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) { - if (err) { - return callback(err); - } - return callback(null, { - cacheOnTimeout: self.options.limits.cacheOnTimeout || false, - render: renderLimit || self.options.limits.render || 0 - }); - }); + var limits = { + cacheOnTimeout: self.options.limits.cacheOnTimeout || false, + render: self.options.limits.render || 0 + }; + + step( + function getTilerLimit() { + var next = this; + self.getTilerRenderLimit(username, function (err, tilerRenderLimit) { + if (err) { + return callback(err); + } + + if (Number.isFinite(tilerRenderLimit)) { + limits.render = tilerRenderLimit; + return callback(null, limits); + } + + return next(); + }); + }, + function getTimeoutLimit() { + self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { + if (err) { + return callback(err); + } + + if (timeoutRenderLimit && timeoutRenderLimit.render) { + if (Number.isFinite(timeoutRenderLimit.render)) { + limits.render = timeoutRenderLimit.render; + } + } + + return callback(null, limits); + }); + } + ); +}; + +UserLimitsApi.prototype.getTilerRenderLimit = function (username, callback) { + this.metadataBackend.getTilerRenderLimit(username, callback); +}; + +UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { + var self = this; + + step( + function isAuthorized() { + var next = this; + + if (!apiKey) { + return next(null, false); + } + + self.metadataBackend.getUserMapKey(username, function (err, userApiKey) { + if (err) { + return next(err); + } + + return next(null, userApiKey === apiKey); + }); + }, + function getUserTimeoutRenderLimits(err, authorized) { + if (err) { + return next(err); + } + self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, this); + }, + function setTilerRenderLimit(err, timeoutRenderLimit) { + if (err) { + return callback(err); + } + return callback(null, timeoutRenderLimit); + } + ); }; diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index af5a775b..340073b5 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { var context = {}; step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, this); + self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index c07f9abc..177322d4 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var context = {}; step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, this); + self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 594ec247..17c66c29 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -114,7 +114,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { function prepareContextLimits(err, _mapConfig) { assert.ifError(err); mapConfig = _mapConfig; - self.userLimitsApi.getRenderLimits(self.owner, this); + self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this); }, function cacheAndReturnMapConfig(err, renderLimits) { self.err = err; diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js new file mode 100644 index 00000000..b595cd0e --- /dev/null +++ b/test/acceptance/user-timeout-limit.js @@ -0,0 +1,82 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); +var testHelper = require('../support/test_helper'); + +var redis = require('redis'); +var keysToDelete; + +function withUserTimeoutRenderLimit(redisClient, user, userTimeoutLimit, callback) { + redisClient.SELECT(5, function(err) { + if (err) { + return callback(err); + } + + var userTimeoutLimitsKey = 'limits:timeout:' + user; + var redisParams = [ + userTimeoutLimitsKey, + 'render', userTimeoutLimit, + 'render_public', userTimeoutLimit + ]; + + redisClient.hmset(redisParams, function (err) { + if (err) { + return callback(err); + } + keysToDelete[userTimeoutLimitsKey] = 5; + return callback(); + }); + }); +} + +function createMapConfig (cartocss) { + return { + version: '1.6.0', + layers: [{ + type: "cartodb", + options: { + sql: [ + 'SELECT', + ' pg_sleep(1),', + ' 1 cartodb_id,', + ' \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' + ].join('\n'), + cartocss: cartocss, + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + }; +} + + + +describe('user timeout limits', function () { + var redisClient = redis.createClient(global.environment.redis.port); + + beforeEach(function() { + keysToDelete = {}; + }); + + afterEach(function (done) { + testHelper.deleteRedisKeys(keysToDelete, done); + }); + + it('layergroup creation works even if test tile is slow', function (done) { + withUserTimeoutRenderLimit(redisClient, 'localhost', 1, function (err) { + if (err) { + return done(err); + } + + var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); + var testClient = new TestClient(mapConfig, 1234); + testClient.getTile(4, 4, 4, {}, function (err, res, tile) { + assert.ok(err, err); + // TODO: check timeout tile + + testClient.drain(done) + }); + }); + }); +}); From 37c970903e5a940d40884488034a402a20a81b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 5 Jul 2017 19:09:14 +0200 Subject: [PATCH 227/402] Avoid uncaught exception when layergroup is not present --- test/support/test-client.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 659d6686..d499e9d5 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -579,8 +579,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { }); }, function finish(err, res, image) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } return callback(err, res, image); } ); From 23aab7a09fa2e7595eadf83bac4d0a9b9b2e4786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 6 Jul 2017 16:23:42 +0200 Subject: [PATCH 228/402] User feature branch of cartodb-redis --- package.json | 2 +- yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 463848c5..dac5756a 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "camshaft": "0.55.6", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", - "cartodb-redis": "0.13.2", + "cartodb-redis": "cartodb/node-cartodb-redis#timeout-limits", "debug": "~2.2.0", "dot": "~1.0.2", "express": "~4.13.3", diff --git a/yarn.lock b/yarn.lock index 8409e6e1..b80967ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -262,9 +262,9 @@ cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" -cartodb-redis@0.13.2: - version "0.13.2" - resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.13.2.tgz#de5214fa5c3ab336c4da978133efa8f908b3691c" +cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: + version "0.13.3" + resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/0c0e5c410243bd4a16e9d94b52821e9d6533eb16" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" @@ -1745,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1770,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: From 32bcf9ca89bf33b72fc00dd90f4a727a74e8f982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 6 Jul 2017 16:24:18 +0200 Subject: [PATCH 229/402] Fix jshint typo --- test/acceptance/user-timeout-limit.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index b595cd0e..cb5fe20e 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -71,11 +71,10 @@ describe('user timeout limits', function () { var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); var testClient = new TestClient(mapConfig, 1234); - testClient.getTile(4, 4, 4, {}, function (err, res, tile) { + testClient.getTile(4, 4, 4, {}, function (err, /* res, tile */) { assert.ok(err, err); // TODO: check timeout tile - - testClient.drain(done) + testClient.drain(done); }); }); }); From ad1506ae976c4976ff6a8b9c6e304d75c61f9b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 6 Jul 2017 16:24:50 +0200 Subject: [PATCH 230/402] Remove empty lines --- test/acceptance/user-timeout-limit.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index cb5fe20e..0c75dce2 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -50,8 +50,6 @@ function createMapConfig (cartocss) { }; } - - describe('user timeout limits', function () { var redisClient = redis.createClient(global.environment.redis.port); From f7a763b6378c4dc9ce1d334f11582f84ec20e1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:09:17 +0200 Subject: [PATCH 231/402] widgets: add aggregation for search results --- lib/cartodb/models/dataview/aggregation.js | 3 ++- package.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index c15f0506..2fb76600 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -253,6 +253,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { var self = this; var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); + var _value = this.aggregation !== 'count' && this.aggregationColumn ? this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ @@ -265,7 +266,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { _searchFiltered: filterCategoriesQueryTpl({ _query: this.query, _column: this.column, - _value: 'count(1)', + _value: _value, _userQuery: _userQuery }) }); diff --git a/package.json b/package.json index c32459f0..f613df30 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "contributors": [ "Simon Tokumine ", "Javi Santana ", - "Sandro Santilli " + "Sandro Santilli ", + "Carlos Matallín " ], "dependencies": { "body-parser": "~1.14.0", From 7fa154c06286d8ae9372f18119a7da6d2fa3b503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:38:15 +0200 Subject: [PATCH 232/402] widgets: add aggregation for search results, specs --- test/acceptance/widgets/ported/aggregation.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/acceptance/widgets/ported/aggregation.js b/test/acceptance/widgets/ported/aggregation.js index 9329f039..62c998a5 100644 --- a/test/acceptance/widgets/ported/aggregation.js +++ b/test/acceptance/widgets/ported/aggregation.js @@ -322,6 +322,25 @@ describe('widgets', function() { }); }); }); + + [adm0name].forEach(function(userQuery) { + it('should search with sum aggregation: ' + userQuery, function(done) { + this.testClient = new TestClient(aggregationSumMapConfig); + this.testClient.widgetSearch('adm0name', userQuery, function (err, res, searchResult) { + assert.ok(!err, err); + assert.ok(searchResult); + assert.equal(searchResult.type, 'aggregation'); + + assert.equal(searchResult.categories.length, 1); + assert.deepEqual( + searchResult.categories, + [{ category:"Argentina", value:28015640 }] + ); + + done(); + }); + }); + }); }); }); From dd934a39139ea07d76709000b827b99143c38f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:44:32 +0200 Subject: [PATCH 233/402] linter --- Makefile | 2 +- lib/cartodb/models/dataview/aggregation.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1913b9e8..a2910a0f 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ config/environments/test.js: config.status--test TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js") TEST_SUITE_UNIT := $(shell find test/unit -name "*.js") TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js") -TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js") +TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance/widgets/ported -name "*.js") test: config/environments/test.js @echo "***tests***" diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 2fb76600..d7710f4f 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -253,7 +253,8 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { var self = this; var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); - var _value = this.aggregation !== 'count' && this.aggregationColumn ? this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; + var _value = this.aggregation !== 'count' && this.aggregationColumn ? + this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ From b1c9dd537e0f07486f3d6a6c8d51fca493cc3a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:47:28 +0200 Subject: [PATCH 234/402] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a2910a0f..1913b9e8 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ config/environments/test.js: config.status--test TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js") TEST_SUITE_UNIT := $(shell find test/unit -name "*.js") TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js") -TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance/widgets/ported -name "*.js") +TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js") test: config/environments/test.js @echo "***tests***" From e36266a80f7d2736bd94e71f334aeba1c90a777b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 10 Jul 2017 12:29:25 +0200 Subject: [PATCH 235/402] Added test to check all aggregation operations work as expected when searching dataviews --- test/acceptance/dataviews/aggregation.js | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 259cd2af..c4b40d44 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -145,4 +145,33 @@ describe('aggregations happy cases', function() { }); }); }); + + var widgetSearchExpects = { + 'count': [ { category: 'other_a', value: 3 } ], + 'sum': [ { category: 'other_a', value: 6 } ], + 'avg': [ { category: 'other_a', value: 2 } ], + 'max': [ { category: 'other_a', value: 3 } ], + 'min': [ { category: 'other_a', value: 1 } ] + }; + + Object.keys(operations_and_values).forEach(function (operation) { + var description = 'should search OTHER category using "' + operation + '"'; + + it(description, function (done) { + this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val')); + this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) { + assert.ifError(err); + + assert.ok(searchResult); + assert.equal(searchResult.type, 'aggregation'); + + assert.equal(searchResult.categories.length, 1); + assert.deepEqual( + searchResult.categories, + widgetSearchExpects[operation] + ); + done(); + }); + }); + }); }); From 5b8ecd3df0a2bc46bf8b5619f3552cd8e9361693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Jul 2017 09:53:12 +0200 Subject: [PATCH 236/402] Release 3.9.6 --- NEWS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 454b9142..b59924bc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,9 @@ # Changelog ## 3.9.6 -Released 2017-mm-dd +Released 2017-07-11 + + - Dataviews: support for aggregation in search results #708 ## 3.9.5 From a35b1e3e8656e9b354e6f092c1099f8e674fe0ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Jul 2017 09:59:30 +0200 Subject: [PATCH 237/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b59924bc..dd6b97f4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.7 +Released 2017-mm-dd + + ## 3.9.6 Released 2017-07-11 diff --git a/package.json b/package.json index 4ed05477..5d6a40df 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.6", + "version": "3.9.7", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 967dca95789595a136ba91592c3ab296629baa85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 12 Jul 2017 15:10:39 +0200 Subject: [PATCH 238/402] Improve readability --- lib/cartodb/models/dataview/histogram.js | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4499dd6c..8d624163 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -159,11 +159,15 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' max({{=it._end}}) AS max_val,', ' min({{=it._start}}) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', - ' min(date_trunc(', - ' \'{{=it._aggregation}}\',', - ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', - ' )) AS start_date,', - ' max(TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', + ' min(', + ' date_trunc(', + ' \'{{=it._aggregation}}\',', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' )', + ' ) AS start_date,', + ' max(', + ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' ) AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -177,15 +181,11 @@ var dateBinsQueryTpl = dot.template([ ' FROM (', ' SELECT', ' ARRAY(', - ' {{?it._aggregation==="quarter"}}', - ' SELECT GENERATE_SERIES(start_date::timestamptz, end_date::timestamptz, \'3 month\'::interval)', - ' {{??}}', ' SELECT GENERATE_SERIES(', ' start_date::timestamptz,', ' end_date::timestamptz,', - ' \'1 {{=it._aggregation}}\'::interval', + ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', - ' {{?}}', ' ) AS bins_array', ' FROM basics', ' ) _cdb_bins_array', @@ -207,10 +207,14 @@ var dateHistogramQueryTpl = dot.template([ ' bins_number', ' )) - 1', ' END AS bin,', - ' min(date_part(', - ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}}::timestamptz', - ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', + ' min(', + ' date_part(', + ' \'epoch\', ', + ' date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', + ' ) AT TIME ZONE \'{{=it._timezone}}\'', + ' )', + ' )::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', From b67f6053e826c3dd047c245af87d7cb4a7e59e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 12 Jul 2017 15:14:41 +0200 Subject: [PATCH 239/402] Should respect first timestamp as bin_start --- lib/cartodb/models/dataview/histogram.js | 6 +-- test/acceptance/dataviews/histogram.js | 48 +++++++++++++++++++++++- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 8d624163..212cf3bb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -562,10 +562,10 @@ function getTimezone(timezone) { function populateBinStart(override, firstRow) { var binStart; - if (override.hasOwnProperty('start')) { - binStart = getBinStart(override); - } else if (firstRow.hasOwnProperty('timestamp')) { + if (firstRow.hasOwnProperty('timestamp')) { binStart = firstRow.timestamp; + } else if (override.hasOwnProperty('start')) { + binStart = getBinStart(override); } else { binStart = firstRow.min; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c5927d52..e33cb2c0 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -173,6 +173,16 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd' } + }, + minute_histogram: { + source: { + id: 'minute-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'minute' + } } }, [ @@ -211,6 +221,18 @@ describe('histogram-dataview for date column type', function() { ") date" ].join(' ') } + }, + { + "id": "minute-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval", + ") date" + ].join(' ') + } } ] ); @@ -282,8 +304,8 @@ describe('histogram-dataview for date column type', function() { it('should override start and end ' + test.desc, function (done) { var params = { - start: 1180659600, // 2007-06-01 01:00:00 - end: 1193792400 // 2007-10-31 01:00:00 + start: 1180659600, // 2007-06-01 01:00:00 UTC => '2007-05-31T21:00:00-04:00' + end: 1193792400 // 2007-10-31 01:00:00 UTC }; this.testClient = new TestClient(mapConfig, 1234); @@ -300,6 +322,28 @@ describe('histogram-dataview for date column type', function() { }); }); + + it('should return same histogram ' + test.desc, function (done) { + var params = { + start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum) + end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum) + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) { + assert.ok(!err, err); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) { + assert.ok(!err, err); + + assert.deepEqual(dataview, filteredDataview); + done(); + }); + }); + }); + + it('should aggregate histogram overriding default timezone to CEST ' + test.desc, function (done) { var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) From c8286233be6393d7ca18ec04c06bc13c5a7c97c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 12 Jul 2017 17:08:55 +0200 Subject: [PATCH 240/402] Do not apply timezone for minute an hour aggregations --- lib/cartodb/models/dataview/histogram.js | 13 ++- test/acceptance/dataviews/histogram.js | 133 +++++++++++++++++++++++ 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 212cf3bb..dd3b2353 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -403,14 +403,14 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _aggregation: _aggregation, _start: getBinStart(override), _end: getBinEnd(override), - _timezone: getTimezone(_timezone) + _timezone: getTimezone(_timezone, _aggregation) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone) + _timezone: getTimezone(_timezone, _aggregation) }); } @@ -429,7 +429,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone) + _timezone: getTimezone(_timezone, _aggregation) }); var histogramSql = [ @@ -551,10 +551,15 @@ function getWidth(override) { return width; } -function getTimezone(timezone) { +function getTimezone(timezone, aggregation) { if (!timezone) { return '0'; } + + if (aggregation === 'hour' || aggregation === 'minute') { + return '0'; + } + var timezoneInHours = Math.ceil(timezone / 3600); return '' + timezoneInHours; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index e33cb2c0..790dbad4 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -547,4 +547,137 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should not apply timezone for a histogram aggregated by minutes', function (done) { + var self = this; + var params = { + timezone: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + + self.testClient.getDataview('minute_histogram', {}, function (err, dataview) { + assert.ifError(err); + self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithTimezone) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithTimezone); + done(); + }); + }); + }); + + it('should filter by "start" & "end" for a histogram aggregated by minutes', function (done) { + var self = this; + var paramsWithFilter = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum) + }; + + var paramsWithTimezone = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum) + timezone: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, dataview) { + assert.ifError(err); + + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, filteredDataview) { + assert.ifError(err); + + assert.deepEqual(dataview, filteredDataview); + + self.testClient.getDataview('minute_histogram', paramsWithTimezone, + function (err, filteredWithTimezoneDataview) { + assert.ifError(err); + + assert.deepEqual(filteredDataview, filteredWithTimezoneDataview); + done(); + }); + }); + }); + }); + + + it('should return an histogram aggregated by days', function (done) { + var self = this; + var paramsWithDailyAgg = { + aggregation: 'day', + }; + + // data: from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggFixture = { + aggregation: 'day', + bin_width: 600, + bins_count: 2, + bins_start: 1171497600, + nulls: 0, + bins: + [{ + bin: 0, + timestamp: 1171497600, + min: 1171583400, + max: 1171583940, + avg: 1171583670, + freq: 10 + }, + { + bin: 1, + timestamp: 1171584000, + min: 1171584000, + max: 1171584600, + avg: 1171584300, + freq: 11 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAgg, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggFixture); + done(); + }); + }); + + it('should return a histogram aggregated by days with timezone', function (done) { + var self = this; + + var paramsWithDailyAggAndTimezone = { + aggregation: 'day', + timezone: '-3600' + }; + + // data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggAndTimezoneFixture = { + aggregation: 'day', + bin_width: 1200, + bins_count: 1, + bins_start: 1171501200, + nulls: 0, + bins: + [{ + bin: 0, + timestamp: 1171501200, + min: 1171583400, + max: 1171584600, + avg: 1171584000, + freq: 21 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndTimezone, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggAndTimezoneFixture); + done(); + }); + }); }); From 21b8655f8568fbdfd8ca187bf22437a821102e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 13 Jul 2017 19:42:22 +0200 Subject: [PATCH 241/402] Return timezone applied or 0 if not present --- lib/cartodb/models/dataview/histogram.js | 18 +++++++++++++++++- test/acceptance/dataviews/histogram.js | 8 +++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dd3b2353..de5a66da 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -491,7 +491,8 @@ Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; - var aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var aggregation = getAggregation(override, this.aggregation); + var timezone = getTimezoneParam(override, this.timezone); var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); @@ -513,6 +514,7 @@ Histogram.prototype.format = function(result, override) { return { aggregation: aggregation, + timezone: timezone, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -522,6 +524,20 @@ Histogram.prototype.format = function(result, override) { }; }; +function getAggregation(override, aggregation) { + return override && override.aggregation ? override.aggregation : aggregation; +} + +function getTimezoneParam(override, timezone) { + if (override && override.timezone) { + return override.timezone; + } + if (timezone) { + return timezone; + } + return 0; +} + function getBinStart(override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 790dbad4..233eed15 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -561,6 +561,8 @@ describe('histogram-dataview for date column type', function() { self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithTimezone) { assert.ifError(err); + assert.notEqual(dataview.timezone, dataviewWithTimezone.timezone); + dataview.timezone = dataviewWithTimezone.timezone; assert.deepEqual(dataview, dataviewWithTimezone); done(); }); @@ -593,7 +595,9 @@ describe('histogram-dataview for date column type', function() { function (err, filteredWithTimezoneDataview) { assert.ifError(err); - assert.deepEqual(filteredDataview, filteredWithTimezoneDataview); + assert.notEqual(filteredWithTimezoneDataview.timezone, filteredDataview.timezone); + filteredWithTimezoneDataview.timezone = filteredDataview.timezone; + assert.deepEqual(filteredWithTimezoneDataview, filteredDataview); done(); }); }); @@ -614,6 +618,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 600, bins_count: 2, bins_start: 1171497600, + timezone: 0, nulls: 0, bins: [{ @@ -660,6 +665,7 @@ describe('histogram-dataview for date column type', function() { bins_count: 1, bins_start: 1171501200, nulls: 0, + timezone: -3600, bins: [{ bin: 0, From 0fffafa1db87ad27e85aac675a39ef138cacedb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 14 Jul 2017 18:22:05 +0200 Subject: [PATCH 242/402] Add timestamp_start in histogram summary to help to build the histogram in frontend side --- lib/cartodb/models/dataview/histogram.js | 23 ++++++++++++++--------- test/acceptance/dataviews/histogram.js | 2 ++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index de5a66da..9d934b61 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -215,13 +215,14 @@ var dateHistogramQueryTpl = dot.template([ ' ) AT TIME ZONE \'{{=it._timezone}}\'', ' )', ' )::numeric AS timestamp,', + ' date_part(\'epoch\', start_date)::numeric AS timestamp_start,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', 'ORDER BY bin' ].join('\n')); @@ -403,14 +404,14 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _aggregation: _aggregation, _start: getBinStart(override), _end: getBinEnd(override), - _timezone: getTimezone(_timezone, _aggregation) + _timezone: parseTimezone(_timezone, _aggregation) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _timezone: parseTimezone(_timezone, _aggregation) }); } @@ -429,7 +430,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _timezone: parseTimezone(_timezone, _aggregation) }); var histogramSql = [ @@ -492,12 +493,13 @@ Histogram.prototype.format = function(result, override) { var buckets = []; var aggregation = getAggregation(override, this.aggregation); - var timezone = getTimezoneParam(override, this.timezone); + var timezone = getTimezone(override, this.timezone); var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); var nulls = 0; var avg; + var timestampStart; if (result.rows.length) { var firstRow = result.rows[0]; @@ -506,15 +508,16 @@ Histogram.prototype.format = function(result, override) { avg = firstRow.avg_val; nulls = firstRow.nulls_count; binsStart = populateBinStart(override, firstRow); - + timestampStart = firstRow.timestamp_start; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'timestamp_start'); }); } return { aggregation: aggregation, timezone: timezone, + timestamp_start: timestampStart, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -528,13 +531,15 @@ function getAggregation(override, aggregation) { return override && override.aggregation ? override.aggregation : aggregation; } -function getTimezoneParam(override, timezone) { +function getTimezone(override, timezone) { if (override && override.timezone) { return override.timezone; } + if (timezone) { return timezone; } + return 0; } @@ -567,7 +572,7 @@ function getWidth(override) { return width; } -function getTimezone(timezone, aggregation) { +function parseTimezone(timezone, aggregation) { if (!timezone) { return '0'; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 233eed15..4efc9345 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -618,6 +618,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 600, bins_count: 2, bins_start: 1171497600, + timestamp_start: 1171497600, timezone: 0, nulls: 0, bins: @@ -664,6 +665,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 1200, bins_count: 1, bins_start: 1171501200, + timestamp_start: 1171497600, nulls: 0, timezone: -3600, bins: From 70b4d5b7fda6d4e4b362d32c64d2f3bb8a8b52e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 14 Jul 2017 18:30:36 +0200 Subject: [PATCH 243/402] replace timezone => offset --- lib/cartodb/backends/dataview.js | 2 +- lib/cartodb/controllers/layergroup.js | 2 +- lib/cartodb/models/dataview/histogram.js | 46 +++++----- test/acceptance/dataviews/histogram.js | 106 +++++++++++------------ test/support/test-client.js | 2 +- 5 files changed, 79 insertions(+), 79 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index e54b3a83..5fe1745a 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -104,7 +104,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) { } function getOverrideParams(params, ownFilter) { - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'timezone'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index fcab20d2..9644aa06 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -89,7 +89,7 @@ LayergroupController.prototype.register = function(app) { 'column_type', // string 'bins', // number 'aggregation', //string - 'timezone', // number + 'offset', // number 'q' // widgets search ]; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index de5a66da..a0814095 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -145,9 +145,9 @@ var dateBasicsQueryTpl = dot.template([ ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', + ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._offset}}\'', ' )) AS start_date,', - ' max({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', + ' max({{=it._column}} AT TIME ZONE \'{{=it._offset}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -162,11 +162,11 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' min(', ' date_trunc(', ' \'{{=it._aggregation}}\',', - ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )', ' ) AS start_date,', ' max(', - ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', @@ -201,7 +201,7 @@ var dateHistogramQueryTpl = dot.template([ ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\',', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', ' bins_array', ' ),', ' bins_number', @@ -212,7 +212,7 @@ var dateHistogramQueryTpl = dot.template([ ' \'epoch\', ', ' date_trunc(', ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', - ' ) AT TIME ZONE \'{{=it._timezone}}\'', + ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', @@ -243,7 +243,7 @@ Time series: options: { column: 'date', // column data type: date aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) - timezone: -7200 // OPTIONAL (UTC offset in seconds) + offset: -7200 // OPTIONAL (UTC offset in seconds) } } */ @@ -257,7 +257,7 @@ function Histogram(query, options, queries) { this.column = options.column; this.bins = options.bins; this.aggregation = options.aggregation; - this.timezone = options.timezone; + this.offset = options.offset; this._columnType = null; } @@ -380,7 +380,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _timezone = override && Number.isFinite(override.timezone) ? override.timezone : this.timezone; + var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; if (!_aggregation) { this.getAutomaticAggregation(psql, function (err, aggregation) { @@ -403,14 +403,14 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _aggregation: _aggregation, _start: getBinStart(override), _end: getBinEnd(override), - _timezone: getTimezone(_timezone, _aggregation) + _offset: getOffset(_offset, _aggregation) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _offset: getOffset(_offset, _aggregation) }); } @@ -429,7 +429,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _offset: getOffset(_offset, _aggregation) }); var histogramSql = [ @@ -492,7 +492,7 @@ Histogram.prototype.format = function(result, override) { var buckets = []; var aggregation = getAggregation(override, this.aggregation); - var timezone = getTimezoneParam(override, this.timezone); + var offset = getOffsetParam(override, this.offset); var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); @@ -514,7 +514,7 @@ Histogram.prototype.format = function(result, override) { return { aggregation: aggregation, - timezone: timezone, + offset: offset, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -528,12 +528,12 @@ function getAggregation(override, aggregation) { return override && override.aggregation ? override.aggregation : aggregation; } -function getTimezoneParam(override, timezone) { - if (override && override.timezone) { - return override.timezone; +function getOffsetParam(override, offset) { + if (override && override.offset) { + return override.offset; } - if (timezone) { - return timezone; + if (offset) { + return offset; } return 0; } @@ -567,8 +567,8 @@ function getWidth(override) { return width; } -function getTimezone(timezone, aggregation) { - if (!timezone) { +function getOffset(offset, aggregation) { + if (!offset) { return '0'; } @@ -576,8 +576,8 @@ function getTimezone(timezone, aggregation) { return '0'; } - var timezoneInHours = Math.ceil(timezone / 3600); - return '' + timezoneInHours; + var offsetInHours = Math.ceil(offset / 3600); + return '' + offsetInHours; } function populateBinStart(override, firstRow) { diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 233eed15..520c0fa7 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -132,7 +132,7 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd', aggregation: 'month', - timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, datetime_histogram_tz: { @@ -143,7 +143,7 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd', aggregation: 'month', - timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, datetime_histogram_automatic: { @@ -238,16 +238,16 @@ describe('histogram-dataview for date column type', function() { ); var dateHistogramsUseCases = [{ - desc: 'supporting timestamp with timezone', + desc: 'supporting timestamp with offset', dataviewId: 'datetime_histogram_tz' }, { - desc: 'supporting timestamp without timezone', + desc: 'supporting timestamp without offset', dataviewId: 'datetime_histogram' }]; dateHistogramsUseCases.forEach(function (test) { it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { - var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes this.testClient = new TestClient(mapConfig, 1234); @@ -260,18 +260,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .utcOffset(OFFSET_EDT_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function(bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .utcOffset(OFFSET_EDT_IN_MINUTES) .add(index, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .utcOffset(OFFSET_EDT_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -344,11 +344,11 @@ describe('histogram-dataview for date column type', function() { }); - it('should aggregate histogram overriding default timezone to CEST ' + test.desc, function (done) { - var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) - var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) { + var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) var params = { - timezone: TIMEZONE_CEST_IN_SECONDS + offset: OFFSET_CEST_IN_SECONDS }; this.testClient = new TestClient(mapConfig, 1234); @@ -361,18 +361,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .utcOffset(OFFSET_CEST_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function (bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .utcOffset(OFFSET_CEST_IN_MINUTES) .add(index, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .utcOffset(OFFSET_CEST_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -384,11 +384,11 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should aggregate histogram overriding default timezone to UTC/GMT ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + it('should aggregate histogram overriding default offset to UTC/GMT ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS + offset: OFFSET_UTC_IN_SECONDS }; this.testClient = new TestClient(mapConfig, 1234); @@ -401,18 +401,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function (bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .add(index, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -425,10 +425,10 @@ describe('histogram-dataview for date column type', function() { }); it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, + offset: OFFSET_UTC_IN_SECONDS, aggregation: 'quarter' }; @@ -442,18 +442,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function (bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .add(index * 3, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -466,9 +466,9 @@ describe('histogram-dataview for date column type', function() { }); it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, + offset: OFFSET_UTC_IN_SECONDS, aggregation: 'quarter', start: 1167609600, // 2007-01-01T00:00:00Z, first bin start end: 1214870399 // 2008-06-30T23:59:59Z, last bin end @@ -487,9 +487,9 @@ describe('histogram-dataview for date column type', function() { }); it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, + offset: OFFSET_UTC_IN_SECONDS, aggregation: 'quarter', start: 1167609600, // 2007-01-01T00:00:00Z, first bin start end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last @@ -548,22 +548,22 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should not apply timezone for a histogram aggregated by minutes', function (done) { + it('should not apply offset for a histogram aggregated by minutes', function (done) { var self = this; var params = { - timezone: '-3600' + offset: '-3600' }; self.testClient = new TestClient(mapConfig, 1234); self.testClient.getDataview('minute_histogram', {}, function (err, dataview) { assert.ifError(err); - self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithTimezone) { + self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithOffset) { assert.ifError(err); - assert.notEqual(dataview.timezone, dataviewWithTimezone.timezone); - dataview.timezone = dataviewWithTimezone.timezone; - assert.deepEqual(dataview, dataviewWithTimezone); + assert.notEqual(dataview.offset, dataviewWithOffset.offset); + dataview.offset = dataviewWithOffset.offset; + assert.deepEqual(dataview, dataviewWithOffset); done(); }); }); @@ -576,10 +576,10 @@ describe('histogram-dataview for date column type', function() { end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum) }; - var paramsWithTimezone = { + var paramsWithOffset = { start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum) - timezone: '-3600' + offset: '-3600' }; self.testClient = new TestClient(mapConfig, 1234); @@ -591,13 +591,13 @@ describe('histogram-dataview for date column type', function() { assert.deepEqual(dataview, filteredDataview); - self.testClient.getDataview('minute_histogram', paramsWithTimezone, - function (err, filteredWithTimezoneDataview) { + self.testClient.getDataview('minute_histogram', paramsWithOffset, + function (err, filteredWithOffsetDataview) { assert.ifError(err); - assert.notEqual(filteredWithTimezoneDataview.timezone, filteredDataview.timezone); - filteredWithTimezoneDataview.timezone = filteredDataview.timezone; - assert.deepEqual(filteredWithTimezoneDataview, filteredDataview); + assert.notEqual(filteredWithOffsetDataview.offset, filteredDataview.offset); + filteredWithOffsetDataview.offset = filteredDataview.offset; + assert.deepEqual(filteredWithOffsetDataview, filteredDataview); done(); }); }); @@ -618,7 +618,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 600, bins_count: 2, bins_start: 1171497600, - timezone: 0, + offset: 0, nulls: 0, bins: [{ @@ -649,23 +649,23 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should return a histogram aggregated by days with timezone', function (done) { + it('should return a histogram aggregated by days with offset', function (done) { var self = this; - var paramsWithDailyAggAndTimezone = { + var paramsWithDailyAggAndOffset = { aggregation: 'day', - timezone: '-3600' + offset: '-3600' }; // data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 - var dataviewWithDailyAggAndTimezoneFixture = { + var dataviewWithDailyAggAndOffsetFixture = { aggregation: 'day', bin_width: 1200, bins_count: 1, bins_start: 1171501200, nulls: 0, - timezone: -3600, + offset: -3600, bins: [{ bin: 0, @@ -679,10 +679,10 @@ describe('histogram-dataview for date column type', function() { }; self.testClient = new TestClient(mapConfig, 1234); - self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndTimezone, function (err, dataview) { + self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndOffset, function (err, dataview) { assert.ifError(err); - assert.deepEqual(dataview, dataviewWithDailyAggAndTimezoneFixture); + assert.deepEqual(dataview, dataviewWithDailyAggAndOffsetFixture); done(); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index dc36929e..76219990 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end', 'aggregation', 'timezone'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } From eccc3597aa68113cf9d69c24caac7f63af8a8f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 17 Jul 2017 19:43:59 +0200 Subject: [PATCH 244/402] Respond with 204 when vector tile is empty --- lib/cartodb/controllers/base.js | 4 +++ test/acceptance/mvt.js | 58 +++++++++++++++++++++++++++++++++ test/support/test-client.js | 16 ++++++--- 3 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 test/acceptance/mvt.js diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index ab6587c2..7cdd0713 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -201,6 +201,10 @@ BaseController.prototype.sendError = function(req, res, err, label) { var statusCode = findStatusCode(err); + if (err.message === 'Tile does not exist' && req.params.format === 'mvt') { + statusCode = 204; + } + debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack); // If a callback was requested, force status to 200 diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js new file mode 100644 index 00000000..17653d1d --- /dev/null +++ b/test/acceptance/mvt.js @@ -0,0 +1,58 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); + +function createMapConfig (sql) { + sql = sql || [ + 'select', + ' *', + 'from', + ' populated_places_simple_reduced', + ].join('\n'); + + return { + version: '1.6.0', + layers: [{ + type: "cartodb", + options: { + sql: sql, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + }; +} + + +describe('mvt', function () { + const testCases = [ + { + desc: 'should get empty mvt with code 204 (no content)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + mapConfig: createMapConfig('select 1 as cartodb_id, null::geometry as the_geom_webmercator') + } + ]; + + testCases.forEach(function (test) { + it(test.desc, done => { + const testClient = new TestClient(test.mapConfig, 1234); + const { z, x, y } = test.coords; + const options = { + format: test.format, + status: 204 + }; + + testClient.getTile(z, x, y, options, (err, res) => { + assert.ifError(err); + + assert.ifError(err); + assert.equal(res.statusCode, 204); + assert.equal(res.body, ''); + testClient.drain(done); + }); + }); + }); +}); \ No newline at end of file diff --git a/test/support/test-client.js b/test/support/test-client.js index 659d6686..72a6eca9 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -525,7 +525,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { }; var expectedResponse = { - status: 200, + status: params.status || 200, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -542,7 +542,12 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { if (isMvt) { request.encoding = 'binary'; - expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; + + if (expectedResponse.status === 200) { + expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; + } else if (expectedResponse.status === 204) { + expectedResponse.headers['Content-Type'] = undefined; + } } var isGeojson = format.match(/geojson$/); @@ -561,15 +566,16 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { assert.response(server, request, expectedResponse, function(res, err) { assert.ifError(err); - var obj; if (isPng) { obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); } else if (isMvt) { - obj = new mapnik.VectorTile(z, x, y); - obj.setDataSync(new Buffer(res.body, 'binary')); + if (res.body) { + obj = new mapnik.VectorTile(z, x, y); + obj.setDataSync(new Buffer(res.body, 'binary')); + } } else { obj = JSON.parse(res.body); From ff1399625568432c2da6eb4e5a54ca42ddb53006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 10:44:27 +0200 Subject: [PATCH 245/402] Add test to check that mvt returns 200 when tile has data --- test/acceptance/mvt.js | 34 +++++++++++++++------------------- test/support/test-client.js | 5 +++++ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 17653d1d..db75bb16 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -1,16 +1,9 @@ require('../support/test_helper'); -var assert = require('../support/assert'); -var TestClient = require('../support/test-client'); - -function createMapConfig (sql) { - sql = sql || [ - 'select', - ' *', - 'from', - ' populated_places_simple_reduced', - ].join('\n'); +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); +function createMapConfig (sql = TestClient.SQL.ONE_POINT) { return { version: '1.6.0', layers: [{ @@ -25,14 +18,21 @@ function createMapConfig (sql) { }; } - describe('mvt', function () { const testCases = [ { desc: 'should get empty mvt with code 204 (no content)', coords: { z: 0, x: 0, y: 0 }, format: 'mvt', - mapConfig: createMapConfig('select 1 as cartodb_id, null::geometry as the_geom_webmercator') + status: 204, + mapConfig: createMapConfig(TestClient.SQL.EMPTY) + }, + { + desc: 'should get mvt tile with code 200 (ok)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + status: 200, + mapConfig: createMapConfig() } ]; @@ -40,17 +40,13 @@ describe('mvt', function () { it(test.desc, done => { const testClient = new TestClient(test.mapConfig, 1234); const { z, x, y } = test.coords; - const options = { - format: test.format, - status: 204 - }; + const { format, status } = test; - testClient.getTile(z, x, y, options, (err, res) => { + testClient.getTile(z, x, y, { format, status }, (err, res) => { assert.ifError(err); assert.ifError(err); - assert.equal(res.statusCode, 204); - assert.equal(res.body, ''); + assert.equal(res.statusCode, test.status); testClient.drain(done); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 72a6eca9..5d0eddff 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -75,6 +75,11 @@ module.exports.CARTOCSS = { ].join('\n') }; +module.exports.SQL = { + EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator', + ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' +} + TestClient.prototype.getWidget = function(widgetName, params, callback) { var self = this; From 0aab434f13fb019e9c23059309fa9941afa6c39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 10:52:24 +0200 Subject: [PATCH 246/402] Remove duplicated assertion --- test/acceptance/mvt.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index db75bb16..6510541b 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -45,7 +45,6 @@ describe('mvt', function () { testClient.getTile(z, x, y, { format, status }, (err, res) => { assert.ifError(err); - assert.ifError(err); assert.equal(res.statusCode, test.status); testClient.drain(done); }); From 446e2d08020fa51c1077a137b3c3bd6e9d94bf9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 11:05:45 +0200 Subject: [PATCH 247/402] Add empty line at the end of file --- test/acceptance/mvt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 6510541b..b1ea4455 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -50,4 +50,4 @@ describe('mvt', function () { }); }); }); -}); \ No newline at end of file +}); From f306c26da61196b7903f94c4ef93a04a6a7fb69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 11:08:39 +0200 Subject: [PATCH 248/402] Update News --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index dd6b97f4..fbce5c7d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.9.7 Released 2017-mm-dd + - Respond with 204 (No content) when vector tile has no data #712 + ## 3.9.6 Released 2017-07-11 From 669707b26c05ab6ef88d35587cd5f0499d345ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 11:56:54 +0200 Subject: [PATCH 249/402] Fix typo --- lib/cartodb/api/user_limits_api.js | 5 ++++- test/acceptance/user-timeout-limit.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 7f244e7d..7e004e31 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -81,10 +81,13 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call }); }, function getUserTimeoutRenderLimits(err, authorized) { + var next = this; + if (err) { return next(err); } - self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, this); + + self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, next); }, function setTilerRenderLimit(err, timeoutRenderLimit) { if (err) { diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index 0c75dce2..c83db965 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -69,7 +69,7 @@ describe('user timeout limits', function () { var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); var testClient = new TestClient(mapConfig, 1234); - testClient.getTile(4, 4, 4, {}, function (err, /* res, tile */) { + testClient.getTile(4, 4, 4, {}, function (err /*, res, tile */) { assert.ok(err, err); // TODO: check timeout tile testClient.drain(done); From a17916488b4760e0784612400f4d2591ef8867f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 17:40:36 +0200 Subject: [PATCH 250/402] Update config to get the right key to enable stats metadate after layergroup creation --- config/environments/development.js.example | 3 +-- config/environments/production.js.example | 2 +- config/environments/staging.js.example | 2 +- config/environments/test.js.example | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 9bc4157b..48d03592 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -324,8 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true - + layerStats: true } }; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index b305deef..ba7fec6f 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -324,7 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: false + layerStats: false } }; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index fa3c7cb6..1792779c 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -324,7 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true + layerStats: true } }; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 465b1774..374467c0 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -318,7 +318,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true + layerStats: true } }; From 87eb5407a8f9dcfa5937fb88e60c8291d9cb2c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 20:50:31 +0200 Subject: [PATCH 251/402] WIP: implement timeout limit for raster --- test/acceptance/user-timeout-limit.js | 118 ++++++++++++++------------ test/support/test-client.js | 55 ++++++++---- test/support/test_helper.js | 19 ++++- 3 files changed, 120 insertions(+), 72 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index c83db965..1d588f74 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -1,48 +1,25 @@ require('../support/test_helper'); -var assert = require('../support/assert'); -var TestClient = require('../support/test-client'); -var testHelper = require('../support/test_helper'); +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); -var redis = require('redis'); -var keysToDelete; +const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; -function withUserTimeoutRenderLimit(redisClient, user, userTimeoutLimit, callback) { - redisClient.SELECT(5, function(err) { - if (err) { - return callback(err); - } +var pointSleepSql = ` + SELECT + pg_sleep(0.5), + 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, + 1 cartodb_id +`; - var userTimeoutLimitsKey = 'limits:timeout:' + user; - var redisParams = [ - userTimeoutLimitsKey, - 'render', userTimeoutLimit, - 'render_public', userTimeoutLimit - ]; - - redisClient.hmset(redisParams, function (err) { - if (err) { - return callback(err); - } - keysToDelete[userTimeoutLimitsKey] = 5; - return callback(); - }); - }); -} - -function createMapConfig (cartocss) { +function createMapConfig (sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS) { return { version: '1.6.0', layers: [{ - type: "cartodb", + type: 'cartodb', options: { - sql: [ - 'SELECT', - ' pg_sleep(1),', - ' 1 cartodb_id,', - ' \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' - ].join('\n'), - cartocss: cartocss, + sql, + cartocss, cartocss_version: '2.3.0', interactivity: 'cartodb_id' } @@ -51,28 +28,63 @@ function createMapConfig (cartocss) { } describe('user timeout limits', function () { - var redisClient = redis.createClient(global.environment.redis.port); + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; - beforeEach(function() { - keysToDelete = {}; + before(function () { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + }); + + after(function () { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + it('layergroup creation works if test tile is fast but tile request fails if they are slow', function (done) { + var testClient = new TestClient(createMapConfig(), 1234); + + testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + assert.ifError(err); + + testClient.getTile(0, 0, 0, {}, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, function (err) { + assert.ifError(err); + testClient.drain(done); + }); + }); + }); + }); }); - afterEach(function (done) { - testHelper.deleteRedisKeys(keysToDelete, done); - }); + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; - it('layergroup creation works even if test tile is slow', function (done) { - withUserTimeoutRenderLimit(redisClient, 'localhost', 1, function (err) { - if (err) { - return done(err); - } + beforeEach(function() { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + }); - var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS); - var testClient = new TestClient(mapConfig, 1234); - testClient.getTile(4, 4, 4, {}, function (err /*, res, tile */) { - assert.ok(err, err); - // TODO: check timeout tile - testClient.drain(done); + afterEach(function() { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + it('layergroup creation works even if test tile is slow', function (done) { + var testClient = new TestClient(createMapConfig(), 1234); + testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + assert.ifError(err); + var params = { + status: 400, + contentType: 'application/json; charset=utf-8' + }; + + testClient.getTile(0, 0, 0, params, function (err, res, tile) { + assert.ifError(err); + + assert.equal(tile.errors[0], 'Render timed out'); + testClient.drain(done); + }); }); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index dbaf466d..80bc9223 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -14,13 +14,13 @@ var helper = require('./test_helper'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); serverOptions.analysis.batch.inlineExecution = true; -var server = new CartodbWindshaft(serverOptions); function TestClient(config, apiKey) { this.mapConfig = isMapConfig(config) ? config : null; this.template = isTemplate(config) ? config : null; this.apiKey = apiKey; this.keysToDelete = {}; + this.server = new CartodbWindshaft(serverOptions); } module.exports = TestClient; @@ -97,7 +97,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -156,7 +156,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -208,7 +208,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -265,7 +265,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call } url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -332,7 +332,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -385,7 +385,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { } url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -441,7 +441,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { params.placeholders = params.placeholders || {}; - assert.response(server, + assert.response(self.server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), method: 'POST', @@ -473,7 +473,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : url; - assert.response(server, + assert.response(self.server, { url: path, method: 'POST', @@ -569,20 +569,22 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; } - assert.response(server, request, expectedResponse, function(res, err) { + if (params.contentType) { + expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + + assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); var obj; if (isPng) { obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } - else if (isMvt) { + } else if (isMvt) { if (res.body) { obj = new mapnik.VectorTile(z, x, y); obj.setDataSync(new Buffer(res.body, 'binary')); } - } - else { + } else { obj = JSON.parse(res.body); } @@ -618,7 +620,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { url += '?' + qs.stringify({api_key: this.apiKey}); } - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -662,7 +664,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -723,7 +725,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { } }; - assert.response(server, request, expectedResponse, function(res, err) { + assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); next(null, res, JSON.parse(res.body)); }); @@ -741,6 +743,10 @@ TestClient.prototype.drain = function(callback) { }; module.exports.getStaticMap = function getStaticMap(templateName, params, callback) { + var self = this; + + self.server = new CartodbWindshaft(serverOptions); + if (!callback) { callback = params; params = null; @@ -771,9 +777,22 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba // this could be removed once named maps are invalidated, otherwise you hits the cache var server = new CartodbWindshaft(serverOptions); - assert.response(server, requestOptions, expectedResponse, function (res, err) { + assert.response(self.server, requestOptions, expectedResponse, function (res, err) { helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() { return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary'))); }); }); }; + +TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) { + const userTimeoutLimitsKey = `limits:timeout:${user}`; + const params = [ + userTimeoutLimitsKey, + 'render', userTimeoutLimit, + 'render_public', userTimeoutLimit + ]; + + this.keysToDelete[userTimeoutLimitsKey] = 5; + + helper.configureMetadata('hmset', params, callback); +} diff --git a/test/support/test_helper.js b/test/support/test_helper.js index de9a6e3c..e8b057c9 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -166,12 +166,29 @@ function rmdirRecursiveSync(dirname) { } } +function configureMetadata(action, params, callback) { + redisClient.SELECT(5, function (err) { + if (err) { + return callback(err); + } + + redisClient[action](params, function (err) { + if (err) { + return callback(err); + } + + return callback(); + }); + }); +} + module.exports = { deleteRedisKeys: deleteRedisKeys, lzma_compress_to_base64: lzma_compress_to_base64, checkNoCache: checkNoCache, checkSurrogateKey: checkSurrogateKey, checkCache: checkCache, - rmdirRecursiveSync: rmdirRecursiveSync + rmdirRecursiveSync: rmdirRecursiveSync, + configureMetadata }; From 8907082a85ef965d2c0de7e4d81592f656273a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 19 Jul 2017 12:24:37 +0200 Subject: [PATCH 252/402] Parse body relying on content-type header --- test/support/test-client.js | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 80bc9223..48ef0eb5 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -575,20 +575,18 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); - var obj; + var body; - if (isPng) { - obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } else if (isMvt) { - if (res.body) { - obj = new mapnik.VectorTile(z, x, y); - obj.setDataSync(new Buffer(res.body, 'binary')); - } - } else { - obj = JSON.parse(res.body); + if (res.headers['content-type'] === 'image/png') { + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + } else if (res.headers['content-type'] === 'application/x-protobuf') { + body = new mapnik.VectorTile(z, x, y); + body.setDataSync(new Buffer(res.body, 'binary')); + } else if (res.headers['content-type'] === 'application/json; charset=utf-8') { + body = JSON.parse(res.body); } - next(null, res, obj); + next(null, res, body); }); }, function finish(err, res, image) { From e9bc0732c0aea7d687864c8c1608497c387dd201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 19 Jul 2017 12:33:57 +0200 Subject: [PATCH 253/402] Use switch statement instead of if ... else if ... else ... --- test/support/test-client.js | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 48ef0eb5..9d1d1d7e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -575,15 +575,22 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); - var body; - if (res.headers['content-type'] === 'image/png') { - body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } else if (res.headers['content-type'] === 'application/x-protobuf') { - body = new mapnik.VectorTile(z, x, y); - body.setDataSync(new Buffer(res.body, 'binary')); - } else if (res.headers['content-type'] === 'application/json; charset=utf-8') { - body = JSON.parse(res.body); + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/x-protobuf': + body = new mapnik.VectorTile(z, x, y); + body.setDataSync(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; } next(null, res, body); From cc8a1df3885f875de772a699bc0bd2da8b4c48f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 12:11:09 +0200 Subject: [PATCH 254/402] Bump turbo-carto version to 0.19.2 --- NEWS.md | 4 ++++ package.json | 2 +- yarn.lock | 10 +++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index fbce5c7d..115c4aa1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,8 +3,12 @@ ## 3.9.7 Released 2017-mm-dd +Bug fixes: - Respond with 204 (No content) when vector tile has no data #712 +Announcements: + - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1) + ## 3.9.6 Released 2017-07-11 diff --git a/package.json b/package.json index 5d6a40df..c73280ec 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "request": "~2.79.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.19.1", + "turbo-carto": "0.19.2", "underscore": "~1.6.0", "windshaft": "3.2.1", "yargs": "~5.0.0" diff --git a/yarn.lock b/yarn.lock index 8409e6e1..a5538d97 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1745,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1770,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -2156,9 +2156,9 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@0.19.1: - version "0.19.1" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.1.tgz#c32af073936a4e8f197dfea918e7441c949d7865" +turbo-carto@^0.19.2: + version "0.19.2" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" From 20003c49cec48ab0c26c08fb2d8904738493f8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 14:53:20 +0200 Subject: [PATCH 255/402] Release 3.9.7 --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 115c4aa1..a0a0aa0a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,13 +1,13 @@ # Changelog ## 3.9.7 -Released 2017-mm-dd +Released 2017-07-20 Bug fixes: - Respond with 204 (No content) when vector tile has no data #712 Announcements: - - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1) + - Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2) ## 3.9.6 From fc3a959da1bbb301ea840cef5bf0880e97ee0780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 15:00:39 +0200 Subject: [PATCH 256/402] Stub next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index a0a0aa0a..cc74a354 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.8 +Released 2017-mm-dd + + ## 3.9.7 Released 2017-07-20 diff --git a/package.json b/package.json index c73280ec..f3fd81a7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.7", + "version": "3.9.8", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 42d05f29ee6b5dae6cf27dcae89bc8a1da716fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 19:47:01 +0200 Subject: [PATCH 257/402] Bump windshaft version to 3.2.2 --- package.json | 2 +- yarn.lock | 66 ++++++++++++++++++++++++---------------------------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/package.json b/package.json index f3fd81a7..d15c9667 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.2.1", + "windshaft": "3.2.2", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index a5538d97..02d88ef6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -53,8 +53,8 @@ ap@~0.2.0: resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" aproba@^1.0.3: - version "1.1.1" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" + version "1.1.2" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" are-we-there-yet@~1.1.2: version "1.1.4" @@ -161,10 +161,6 @@ browser-stdout@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" -buffer-shims@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" - buffer-writer@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" @@ -209,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -235,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -899,7 +895,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1: +inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -1348,8 +1344,8 @@ nock@~2.11.0: propagate "0.3.x" node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: - version "0.6.34" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" + version "0.6.36" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1388,8 +1384,8 @@ normalize-package-data@^2.3.2: validate-npm-package-license "^3.0.1" npmlog@^4.0.2: - version "4.1.0" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.0.tgz#dc59bee85f64f00ed424efb2af0783df25d1c0b5" + version "4.1.2" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" dependencies: are-we-there-yet "~1.1.2" console-control-strings "~1.1.0" @@ -1708,15 +1704,15 @@ readable-stream@1.1, readable-stream@~1.1.9: string_decoder "~0.10.x" readable-stream@^2.0.6, readable-stream@^2.1.4: - version "2.2.9" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" + version "2.3.3" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" dependencies: - buffer-shims "~1.0.0" core-util-is "~1.0.0" - inherits "~2.0.1" + inherits "~2.0.3" isarray "~1.0.0" process-nextick-args "~1.0.6" - string_decoder "~1.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.0.3" util-deprecate "~1.0.1" readable-stream@~1.0.2: @@ -1827,9 +1823,9 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" +safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" safe-json-stringify@~1: version "1.0.4" @@ -2029,11 +2025,11 @@ string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" -string_decoder@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" +string_decoder@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" dependencies: - buffer-shims "~1.0.0" + safe-buffer "~5.1.0" stringstream@~0.0.4: version "0.0.5" @@ -2107,15 +2103,15 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: - version "2.3.1-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3": + version "2.3.1-cdb3" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2156,7 +2152,7 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@^0.19.2: +turbo-carto@0.19.2: version "0.19.2" resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd" dependencies: @@ -2278,9 +2274,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.1.tgz#50a3afa6562315dd9e65e411660970e118f36c19" +windshaft@3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.2.tgz#7afb9d8fd8bba1bf02d39c06e8bbe5a451aad953" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2297,7 +2293,7 @@ windshaft@3.2.1: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb2 + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb3 tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2 torque.js "~2.11.0" underscore "~1.6.0" From 106b9a64b23b0090c8ae7222d168de1a4c4bf591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 21 Jul 2017 10:39:33 +0200 Subject: [PATCH 258/402] Update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index cc74a354..6a591fff 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.9.8 Released 2017-mm-dd + - Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2). + ## 3.9.7 Released 2017-07-20 From 35b9448e9a18f976459a762fee5ad0733b99c0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 21 Jul 2017 10:58:37 +0200 Subject: [PATCH 259/402] Release 3.9.8 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6a591fff..87124867 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.9.8 -Released 2017-mm-dd +Released 2017-07-21 - Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2). From 52da3bfa55f8fc07d8215f6bd3b6191954b2c973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 21 Jul 2017 11:05:51 +0200 Subject: [PATCH 260/402] Stub next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 87124867..a3228af1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.9 +Released 2017-mm-dd + + ## 3.9.8 Released 2017-07-21 diff --git a/package.json b/package.json index d15c9667..08dc20cf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.8", + "version": "3.9.9", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From eee4fc815ebad60a948e0112e2641691987b8c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:11:56 +0200 Subject: [PATCH 261/402] Do not expose database error details --- lib/cartodb/models/dataview/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index f8eb1027..7c3dbe6f 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -9,9 +9,9 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { var self = this; this.sql(psql, override, function(err, query) { psql.query(query, function(err, result) { - if (err) { - return callback(err, result); + var error = new Error(err.message); + return callback(error, result); } result = self.format(result, override); From 444595d49d41efcc0b74517357b7048149727bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:14:04 +0200 Subject: [PATCH 262/402] Rely on windshaft's repo branch --- package.json | 2 +- yarn.lock | 92 ++++++++++++++++++++++++++-------------------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/package.json b/package.json index 19c00700..093a6948 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.2.2", + "windshaft": "cartodb/windshaft#response-timeout-limit", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index c6a468fa..14962708 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,7 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -239,6 +231,14 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -1741,32 +1741,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.81.0: +request@2.x, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1793,6 +1768,31 @@ request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -2103,17 +2103,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3": - version "2.3.1-cdb3" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23" +tilelive-bridge@cartodb/tilelive-bridge#query-timeout: + version "2.3.1-cdb2" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/38d5d82355aa9eef68c29b7d7a33b60a3ae394f7" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#custom-timeout: version "0.6.18-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/d9bfe27a4d5120be92e71337474135e2877e9cd4" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2274,9 +2274,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.2.2: +windshaft@cartodb/windshaft#response-timeout-limit: version "3.2.2" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.2.tgz#7afb9d8fd8bba1bf02d39c06e8bbe5a451aad953" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/21ebe5f69b5b6215ec2f93088d8eee436f6b81d5" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2293,8 +2293,8 @@ windshaft@3.2.2: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb3 - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2 + tilelive-bridge cartodb/tilelive-bridge#query-timeout + tilelive-mapnik cartodb/tilelive-mapnik#custom-timeout torque.js "~2.11.0" underscore "~1.6.0" From 526e850f26b364be02a316401b05bc6a153b2b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:15:43 +0200 Subject: [PATCH 263/402] Add method to set statements timeout for user's role and database --- test/support/test-client.js | 73 ++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 9d1d1d7e..411cdf3b 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -3,7 +3,8 @@ var qs = require('querystring'); var step = require('step'); var urlParser = require('url'); - +var PSQL = require('cartodb-psql'); +var _ = require('underscore'); var mapnik = require('windshaft').mapnik; var LayergroupToken = require('./layergroup-token'); @@ -72,6 +73,34 @@ module.exports.CARTOCSS = { ' line-width: 0.5;', ' line-opacity: 1;', '}' + ].join('\n'), + + TORQUE: [ + 'Map {', + ' -torque-frame-count: 256;', + ' -torque-animation-duration: 30;', + ' -torque-time-attribute: "cartodb_id";', + ' -torque-aggregation-function: "count(1)";', + ' -torque-resolution: 4;', + ' -torque-data-aggregation: linear;', + '}', + '#layer {', + ' marker-width: 7;', + ' marker-fill: #FFB927;', + ' marker-fill-opacity: 0.9;', + ' marker-line-width: 1;', + ' marker-line-color: #FFF;', + ' marker-line-opacity: 1;', + ' comp-op: lighter;', + '}', + '#layer[frame-offset=1] {', + ' marker-width: 9;', + ' marker-fill-opacity: 0.45;', + '}', + '#layer[frame-offset=2] {', + ' marker-width: 11;', + ' marker-fill-opacity: 0.225;', + '}' ].join('\n') }; @@ -800,4 +829,44 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi this.keysToDelete[userTimeoutLimitsKey] = 5; helper.configureMetadata('hmset', params, callback); -} +}; + +TestClient.prototype.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { + const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; + const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + + const psql = new PSQL({ + user: 'postgres', + dbname: dbname, + host: global.environment.postgres.host, + port: global.environment.postgres.port + }); + + step( + function setTimeoutToUserRole (err, params) { + const next = this; + + const timeoutQuery = `ALTER ROLE \"${role}\" SET statement_timeout to ${userTimeoutLimit}`; + psql.query(timeoutQuery, function (err) { + if (err) { + return next(err); + } + next(null, params, psql); + }); + }, + function setTimeoutToDatabase (err, params, psql) { + assert.ifError(err); + + const timeoutQuery = `ALTER DATABASE \"${dbname}\" SET statement_timeout to ${userTimeoutLimit}`; + + psql.query(timeoutQuery, this); + }, + function finish (err) { + if (err) { + return callback(err); + } + + callback(); + } + ); +}; From 788b2f06830d093554edbed6d681bc4416ab9d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 25 Jul 2017 19:16:37 +0200 Subject: [PATCH 264/402] Implement test to validate response limits work as expected --- test/acceptance/user-timeout-limit.js | 324 ++++++++++++++++++++++---- 1 file changed, 278 insertions(+), 46 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index 1d588f74..6d30785a 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -5,87 +5,319 @@ const TestClient = require('../support/test-client'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; -var pointSleepSql = ` +const pointSleepSql = ` SELECT pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id `; -function createMapConfig (sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS) { - return { - version: '1.6.0', - layers: [{ - type: 'cartodb', - options: { - sql, - cartocss, - cartocss_version: '2.3.0', - interactivity: 'cartodb_id' +const validationPointSleepSql = ` + SELECT + pg_sleep(0.5), + 'SRID=3857;POINT(-180 90)'::geometry the_geom_webmercator, + 1 cartodb_id +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = pointSleepSql, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: "a0" + }, + cartocss, + cartocss_version, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql } - }] - }; -} + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); -describe('user timeout limits', function () { - describe('with onTileErrorStrategy ENABLED', function () { - let onTileErrorStrategy; - - before(function () { - onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; - global.environment.enabledFeatures.onTileErrorStrategy = true; +describe('user timeout limit', function () { + describe.skip('map instantiation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); }); - after(function () { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + afterEach(function (done) { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); - it('layergroup creation works if test tile is fast but tile request fails if they are slow', function (done) { - var testClient = new TestClient(createMapConfig(), 1234); + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; - testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.ifError(err); - testClient.getTile(0, 0, 0, {}, function (err, res, tile) { - assert.ifError(err); - - assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, function (err) { - assert.ifError(err); - testClient.drain(done); - }); + assert.deepEqual(timeoutError, { + errors: ['Render timed out'], + errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] }); + + done(); }); }); }); - describe('with onTileErrorStrategy DISABLED', function() { - var onTileErrorStrategy; - - beforeEach(function() { - onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; - global.environment.enabledFeatures.onTileErrorStrategy = false; + describe('user torque timeout limits', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); }); - afterEach(function() { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); - it('layergroup creation works even if test tile is slow', function (done) { - var testClient = new TestClient(createMapConfig(), 1234); - testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) { + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.ifError(err); + + assert.deepEqual(timeoutError, { + errors: ["TorqueRenderer: canceling statement due to statement timeout"], + errors_with_context: [{ + "type": "layer", + "message": "TorqueRenderer: canceling statement due to statement timeout", + "layer": { "id": "torque-layer0", "index": 0, "type": "torque" } + }] + }); + + done(); + }); + }); + }); + + describe('dataview', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but dataview request fails due to statement timeout', function (done) { + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getDataview('count', params, (err, dataview) => { + assert.ifError(err); + + assert.deepEqual(dataview, { + errors: ['canceling statement due to statement timeout'], + errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] + }); + + done(); + }); + }); + }); + + + describe('raster', function () { + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function () { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + }); + + afterEach(function () { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + + it('layergroup creation works but tile request fails due to render timeout', function (done) { + this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + }); + + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; + + beforeEach(function() { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + }); + + afterEach(function() { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + }); + + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works even if render tile is slow', function (done) { var params = { status: 400, contentType: 'application/json; charset=utf-8' }; - testClient.getTile(0, 0, 0, params, function (err, res, tile) { + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); assert.equal(tile.errors[0], 'Render timed out'); - testClient.drain(done); + done(); }); }); }); }); + + describe('user vector timeout limits', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + status: 400 + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['Render timed out'], + errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + }); + + done(); + }); + }); + }); + + describe('user interativity timeout limits', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) { + const params = { + layers: 'mapnik', + format: 'grid.json', + status: 400 + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['Render timed out'], + errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + }); + + done(); + }); + }); + }); }); + From 058f19ab3646b3dcd7c02f7996b159fc123b590e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 26 Jul 2017 13:25:37 +0200 Subject: [PATCH 265/402] Fix skipped test --- test/acceptance/user-timeout-limit.js | 33 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index 6d30785a..f4a3877e 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -9,14 +9,17 @@ const pointSleepSql = ` SELECT pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, - 1 cartodb_id + 1 cartodb_id, + 2 value `; +// during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout const validationPointSleepSql = ` SELECT - pg_sleep(0.5), - 'SRID=3857;POINT(-180 90)'::geometry the_geom_webmercator, - 1 cartodb_id + pg_sleep(1), + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 value `; const createMapConfig = ({ @@ -26,18 +29,20 @@ const createMapConfig = ({ cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', interactivity = 'cartodb_id', - countBy = 'cartodb_id' + countBy = 'cartodb_id', + attributes = { id: 'cartodb_id', columns: ['value'] }, } = {}) => ({ version, layers: [{ type, options: { source: { - id: "a0" + id: 'a0' }, cartocss, cartocss_version, - interactivity + interactivity, + attributes } }], analyses: [ @@ -64,7 +69,7 @@ const createMapConfig = ({ }); describe('user timeout limit', function () { - describe.skip('map instantiation', function () { + describe('map instantiation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); @@ -92,8 +97,16 @@ describe('user timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ['Render timed out'], - errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + errors: ["Render timed out"], + errors_with_context: [{ + type: "layer", + message: "Render timed out", + layer: { + id: "layer0", + index: 0, + type: "mapnik" + } + }] }); done(); From 768d06c582098ecfd4b6de9430856c57dd6fa7a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 26 Jul 2017 18:35:40 +0200 Subject: [PATCH 266/402] Assert layer params is defined but mapnik layers --- test/support/test-client.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 411cdf3b..961914d7 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -16,6 +16,14 @@ var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); serverOptions.analysis.batch.inlineExecution = true; +const MAPNIK_SUPPORTED_FORMATS = { + 'png': true, + 'png32': true, + 'grid.json': true, + 'geojson': true, + 'mvt': true +} + function TestClient(config, apiKey) { this.mapConfig = isMapConfig(config) ? config : null; this.template = isTemplate(config) ? config : null; @@ -543,6 +551,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { var format = params.format || 'png'; + if (layers === undefined && !MAPNIK_SUPPORTED_FORMATS[format]) { + throw new Error(`Missing layer filter while fetching ${format} tile, review params argument`); + } + url += [z,x,y].join('/'); url += '.' + format; From 522fc79d71dfa4f70baee5a9ca53713eb791ff3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 12:46:38 +0200 Subject: [PATCH 267/402] Do not remove redis' keys if layergroup was not created --- test/support/test-client.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 961914d7..c78d6e04 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -441,9 +441,16 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }, function finish(err, dataview) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - return callback(err, dataview); + if (err) { + return callback(err); + } + + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(null, dataview); } ); }; From 9e491e7e9a44d2134e2b896fc47b67062c37bb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 16:29:36 +0200 Subject: [PATCH 268/402] Fix tests names --- test/acceptance/user-timeout-limit.js | 4 ++-- test/support/test-client.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index f4a3877e..d3de7164 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -272,7 +272,7 @@ describe('user timeout limit', function () { }); }); - describe('user vector timeout limits', function () { + describe('vector', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); @@ -302,7 +302,7 @@ describe('user timeout limit', function () { }); }); - describe('user interativity timeout limits', function () { + describe('interativity', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); diff --git a/test/support/test-client.js b/test/support/test-client.js index c78d6e04..80869152 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -850,7 +850,7 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.prototype.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { +TestClient.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) From eba97a41e5b912c4a87793583aa1d306a043456f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 16:30:57 +0200 Subject: [PATCH 269/402] Going green, solves issue with role timeout in database --- test/acceptance/user-timeout-limit.js | 108 ++++++++++++-------------- 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index d3de7164..dca4ece7 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -7,10 +7,10 @@ const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.pn const pointSleepSql = ` SELECT - pg_sleep(0.5), + pg_sleep(1), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, - 2 value + 2 val `; // during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout @@ -19,7 +19,7 @@ const validationPointSleepSql = ` pg_sleep(1), ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, 1 cartodb_id, - 2 value + 2 val `; const createMapConfig = ({ @@ -30,7 +30,7 @@ const createMapConfig = ({ cartocss_version = '2.3.0', interactivity = 'cartodb_id', countBy = 'cartodb_id', - attributes = { id: 'cartodb_id', columns: ['value'] }, + attributes } = {}) => ({ version, layers: [{ @@ -68,7 +68,15 @@ const createMapConfig = ({ } }); -describe('user timeout limit', function () { +describe.only('user timeout limit', function () { + before(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 900, done); + }); + + after(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + describe('map instantiation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); @@ -114,64 +122,14 @@ describe('user timeout limit', function () { }); }); - describe('user torque timeout limits', function () { - beforeEach(function (done) { - const mapconfig = createMapConfig({ - type: 'torque', - cartocss: TestClient.CARTOCSS.TORQUE - }); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); - }); - - afterEach(function (done) { - this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); - } - this.testClient.drain(done); - }); - }); - - it('layergroup creation fails due to statement timeout', function (done) { - const expectedResponse = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { - assert.ifError(err); - - assert.deepEqual(timeoutError, { - errors: ["TorqueRenderer: canceling statement due to statement timeout"], - errors_with_context: [{ - "type": "layer", - "message": "TorqueRenderer: canceling statement due to statement timeout", - "layer": { "id": "torque-layer0", "index": 0, "type": "torque" } - }] - }); - - done(); - }); - }); - }); - describe('dataview', function () { - beforeEach(function (done) { + beforeEach(function () { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserDatabaseTimeoutLimit('localhost', 50, done); }); afterEach(function (done) { - this.testClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); - } - this.testClient.drain(done); - }); + this.testClient.drain(done); }); it('layergroup creation works but dataview request fails due to statement timeout', function (done) { @@ -197,6 +155,42 @@ describe('user timeout limit', function () { }); }); + describe('torque', function () { + beforeEach(function () { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: ["TorqueRenderer: canceling statement due to statement timeout"], + errors_with_context: [{ + type: "layer", + message: "TorqueRenderer: canceling statement due to statement timeout", + layer: { id: 'torque-layer0', index: 0, type: "torque" } + }] + }); + + done(); + }); + }); + }); + describe('raster', function () { describe('with onTileErrorStrategy ENABLED', function () { From 664a4e673a1aec5d561d7a156a1c3794382afbd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 17:08:29 +0200 Subject: [PATCH 270/402] Add missing after-each hooks and merged duplicates --- test/acceptance/user-timeout-limit.js | 50 +++++++++++++++++---------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-timeout-limit.js index dca4ece7..87c07b76 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-timeout-limit.js @@ -68,7 +68,7 @@ const createMapConfig = ({ } }); -describe.only('user timeout limit', function () { +describe('user timeout limit', function () { before(function (done) { TestClient.setUserDatabaseTimeoutLimit('localhost', 900, done); }); @@ -196,23 +196,24 @@ describe.only('user timeout limit', function () { describe('with onTileErrorStrategy ENABLED', function () { let onTileErrorStrategy; - beforeEach(function () { + beforeEach(function (done) { onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; global.environment.enabledFeatures.onTileErrorStrategy = true; - }); - afterEach(function () { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - }); - - beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); }); afterEach(function (done) { - this.testClient.drain(done); + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); @@ -231,23 +232,24 @@ describe.only('user timeout limit', function () { describe('with onTileErrorStrategy DISABLED', function() { var onTileErrorStrategy; - beforeEach(function() { + beforeEach(function (done) { onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; global.environment.enabledFeatures.onTileErrorStrategy = false; - }); - afterEach(function() { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - }); - - beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); }); afterEach(function (done) { - this.testClient.drain(done); + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works even if render tile is slow', function (done) { @@ -274,7 +276,12 @@ describe.only('user timeout limit', function () { }); afterEach(function (done) { - this.testClient.drain(done); + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works but vector tile request fails due to render timeout', function (done) { @@ -304,7 +311,12 @@ describe.only('user timeout limit', function () { }); afterEach(function (done) { - this.testClient.drain(done); + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) { From 33e77a42f21f3e5e3243a05f122ee91ea9c5a7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 27 Jul 2017 18:50:27 +0200 Subject: [PATCH 271/402] Separate user timeout suites between render and database --- .../acceptance/user-database-timeout-limit.js | 133 ++++++++++++++++++ ...-limit.js => user-render-timeout-limit.js} | 82 +---------- test/support/test-client.js | 36 ++--- 3 files changed, 147 insertions(+), 104 deletions(-) create mode 100644 test/acceptance/user-database-timeout-limit.js rename test/acceptance/{user-timeout-limit.js => user-render-timeout-limit.js} (75%) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js new file mode 100644 index 00000000..32976b37 --- /dev/null +++ b/test/acceptance/user-database-timeout-limit.js @@ -0,0 +1,133 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); + +const pointSleepSql = ` + SELECT + pg_sleep(1), + 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = pointSleepSql, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); + +describe('user database timeout limit', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + describe('dataview', function () { + beforeEach(function () { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation works but dataview request fails due to statement timeout', function (done) { + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getDataview('count', params, (err, dataview) => { + assert.ifError(err); + + assert.deepEqual(dataview, { + errors: ['canceling statement due to statement timeout'], + errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] + }); + + done(); + }); + }); + }); + + describe('torque', function () { + beforeEach(function () { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: ["TorqueRenderer: canceling statement due to statement timeout"], + errors_with_context: [{ + type: "layer", + message: "TorqueRenderer: canceling statement due to statement timeout", + layer: { id: 'torque-layer0', index: 0, type: "torque" } + }] + }); + + done(); + }); + }); + }); +}); diff --git a/test/acceptance/user-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js similarity index 75% rename from test/acceptance/user-timeout-limit.js rename to test/acceptance/user-render-timeout-limit.js index 87c07b76..8d7e9a6e 100644 --- a/test/acceptance/user-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -68,16 +68,8 @@ const createMapConfig = ({ } }); -describe('user timeout limit', function () { - before(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 900, done); - }); - - after(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); - }); - - describe('map instantiation', function () { +describe('user render timeout limit', function () { + describe('map instantiation => validation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); @@ -122,76 +114,6 @@ describe('user timeout limit', function () { }); }); - describe('dataview', function () { - beforeEach(function () { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - }); - - afterEach(function (done) { - this.testClient.drain(done); - }); - - it('layergroup creation works but dataview request fails due to statement timeout', function (done) { - const params = { - response: { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - } - }; - - this.testClient.getDataview('count', params, (err, dataview) => { - assert.ifError(err); - - assert.deepEqual(dataview, { - errors: ['canceling statement due to statement timeout'], - errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] - }); - - done(); - }); - }); - }); - - describe('torque', function () { - beforeEach(function () { - const mapconfig = createMapConfig({ - type: 'torque', - cartocss: TestClient.CARTOCSS.TORQUE - }); - this.testClient = new TestClient(mapconfig, 1234); - }); - - afterEach(function (done) { - this.testClient.drain(done); - }); - - it('layergroup creation fails due to statement timeout', function (done) { - const expectedResponse = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { - assert.deepEqual(timeoutError, { - errors: ["TorqueRenderer: canceling statement due to statement timeout"], - errors_with_context: [{ - type: "layer", - message: "TorqueRenderer: canceling statement due to statement timeout", - layer: { id: 'torque-layer0', index: 0, type: "torque" } - }] - }); - - done(); - }); - }); - }); - - describe('raster', function () { describe('with onTileErrorStrategy ENABLED', function () { let onTileErrorStrategy; diff --git a/test/support/test-client.js b/test/support/test-client.js index 80869152..d03fef2e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -850,9 +850,10 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callback) { +TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + const publicUser = global.environment.postgres.user; const psql = new PSQL({ user: 'postgres', @@ -862,30 +863,17 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, userTimeoutLimit, callb }); step( - function setTimeoutToUserRole (err, params) { - const next = this; + function configureTimeouts () { + const timeoutSQLs = [ + `ALTER ROLE \"${publicUser}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER ROLE \"${role}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER DATABASE \"${dbname}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` + ]; - const timeoutQuery = `ALTER ROLE \"${role}\" SET statement_timeout to ${userTimeoutLimit}`; - psql.query(timeoutQuery, function (err) { - if (err) { - return next(err); - } - next(null, params, psql); - }); + const group = this.group(); + + timeoutSQLs.forEach(sql => psql.query(sql, group())); }, - function setTimeoutToDatabase (err, params, psql) { - assert.ifError(err); - - const timeoutQuery = `ALTER DATABASE \"${dbname}\" SET statement_timeout to ${userTimeoutLimit}`; - - psql.query(timeoutQuery, this); - }, - function finish (err) { - if (err) { - return callback(err); - } - - callback(); - } + callback ); }; From b023a155b7782faa52aae6b430f88979f3d28b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 13:21:17 +0200 Subject: [PATCH 272/402] Be more accurate with timeouts and pg_sleep --- test/acceptance/user-database-timeout-limit.js | 4 ++-- test/acceptance/user-render-timeout-limit.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 32976b37..c59b26df 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -5,7 +5,7 @@ const TestClient = require('../support/test-client'); const pointSleepSql = ` SELECT - pg_sleep(1), + pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, 2 val @@ -55,7 +55,7 @@ const createMapConfig = ({ describe('user database timeout limit', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 50, done); + TestClient.setUserDatabaseTimeoutLimit('localhost', 300, done); }); afterEach(function (done) { diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 8d7e9a6e..0119a1b9 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -16,7 +16,7 @@ const pointSleepSql = ` // during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout const validationPointSleepSql = ` SELECT - pg_sleep(1), + pg_sleep(0.5), ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, 1 cartodb_id, 2 val From 815eac5a48987a521da7e2e215f54f3ab7727b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 13:22:16 +0200 Subject: [PATCH 273/402] Add hook to refresh all connections in the pool --- test/support/test_helper.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/support/test_helper.js b/test/support/test_helper.js index e8b057c9..fc9fcb34 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -14,6 +14,7 @@ var lzmaWorker = new LZMA(); var redis = require('redis'); var nock = require('nock'); var log4js = require('log4js'); +var pg = require('pg'); // set environment specific variables global.environment = require(__dirname + '/../../config/environments/test'); @@ -127,6 +128,12 @@ afterEach(function(done) { }); }); + +afterEach(function () { + // TODO: this method will be replaced by psql.end + pg.end(); +}); + function deleteRedisKeys(keysToDelete, callback) { if (Object.keys(keysToDelete).length === 0) { From cca570e832ea917eae7532467229f7fa0295c7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 13:23:13 +0200 Subject: [PATCH 274/402] Uncomment DB and role configuration --- test/support/test-client.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index d03fef2e..1d66d2b6 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -852,8 +852,9 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; - const role = _.template(global.environment.postgres_auth_user, { user_id: 1 }) - const publicUser = global.environment.postgres.user; + const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) + const publicuser = global.environment.postgres.user; const psql = new PSQL({ user: 'postgres', @@ -865,9 +866,9 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) step( function configureTimeouts () { const timeoutSQLs = [ - `ALTER ROLE \"${publicUser}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, - `ALTER ROLE \"${role}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, - `ALTER DATABASE \"${dbname}\" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` + // `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + // `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` ]; const group = this.group(); From c3e137bb0029538c8bd4913f010c2057477884d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 16:02:54 +0200 Subject: [PATCH 275/402] Update dependency --- yarn.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index 14962708..d7c06ac0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2113,7 +2113,7 @@ tilelive-bridge@cartodb/tilelive-bridge#query-timeout: tilelive-mapnik@cartodb/tilelive-mapnik#custom-timeout: version "0.6.18-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/d9bfe27a4d5120be92e71337474135e2877e9cd4" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/f9042e5eab92e26fb9e2a6ace65af92cc176a218" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" From 6936107b683d7a9441a07870a0ca8c96f80bc66a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 28 Jul 2017 16:04:11 +0200 Subject: [PATCH 276/402] Adjust pg_sleep to timeout --- test/acceptance/user-render-timeout-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 0119a1b9..8c167d4f 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -7,7 +7,7 @@ const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.pn const pointSleepSql = ` SELECT - pg_sleep(1), + pg_sleep(0.5), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, 2 val From f2e043b0631d97d7bcbd65cc5a11ce8d095a2790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 17:56:58 +0200 Subject: [PATCH 277/402] Do not expose database error info --- lib/cartodb/controllers/base.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 7cdd0713..30883e04 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -229,6 +229,18 @@ function stripConnectionInfo(message) { .replace(/is the server.*encountered/im, 'encountered'); } +var ERROR_INFO_TO_EXOSE = { + message: true, + layer: true, + type: true, + analysis: true, + subtype: true +}; + +function shouldBeExposed (prop) { + return !!ERROR_INFO_TO_EXOSE[prop]; +} + function errorMessage(err) { // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 var message = (_.isString(err) ? err : err.message) || 'Unknown error'; @@ -247,7 +259,7 @@ function errorMessageWithContext(err) { for (var prop in err) { // type & message are properties from Error's prototype and will be skipped - if (err.hasOwnProperty(prop)) { + if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) { error[prop] = err[prop]; } } From 398369a5c7ef301cb11975059391e7c9731cf90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 17:58:33 +0200 Subject: [PATCH 278/402] Do not rely on expected defaults headers --- test/acceptance/mvt.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index b1ea4455..b5a59cd8 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -24,14 +24,24 @@ describe('mvt', function () { desc: 'should get empty mvt with code 204 (no content)', coords: { z: 0, x: 0, y: 0 }, format: 'mvt', - status: 204, + response: { + status: 204, + headers: { + 'Content-Type': undefined + } + }, mapConfig: createMapConfig(TestClient.SQL.EMPTY) }, { desc: 'should get mvt tile with code 200 (ok)', coords: { z: 0, x: 0, y: 0 }, format: 'mvt', - status: 200, + response: { + status: 200, + headers: { + 'Content-Type': 'application/x-protobuf' + } + }, mapConfig: createMapConfig() } ]; @@ -40,12 +50,12 @@ describe('mvt', function () { it(test.desc, done => { const testClient = new TestClient(test.mapConfig, 1234); const { z, x, y } = test.coords; - const { format, status } = test; + const { format, response } = test; - testClient.getTile(z, x, y, { format, status }, (err, res) => { + testClient.getTile(z, x, y, { format, response }, (err, res) => { assert.ifError(err); - assert.equal(res.statusCode, test.status); + assert.equal(res.statusCode, test.response.status); testClient.drain(done); }); }); From 64f19b65ec411b4d7f4e13c15513b85a17b8dbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:01:19 +0200 Subject: [PATCH 279/402] Remove attributes param --- test/acceptance/user-render-timeout-limit.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 8c167d4f..a44c8793 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -29,8 +29,7 @@ const createMapConfig = ({ cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', interactivity = 'cartodb_id', - countBy = 'cartodb_id', - attributes + countBy = 'cartodb_id' } = {}) => ({ version, layers: [{ @@ -41,8 +40,7 @@ const createMapConfig = ({ }, cartocss, cartocss_version, - interactivity, - attributes + interactivity } }], analyses: [ From 664db4b5cfefbffc9886a7cfd16ac25e44284543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:07:44 +0200 Subject: [PATCH 280/402] Pass proper param to check content-type and status --- test/acceptance/user-render-timeout-limit.js | 25 +++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index a44c8793..cd5b03a7 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -136,7 +136,6 @@ describe('user render timeout limit', function () { }); }); - it('layergroup creation works but tile request fails due to render timeout', function (done) { this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { assert.ifError(err); @@ -172,10 +171,14 @@ describe('user render timeout limit', function () { }); }); - it('layergroup creation works even if render tile is slow', function (done) { + it('layergroup creation works and render tile fails', function (done) { var params = { - status: 400, - contentType: 'application/json; charset=utf-8' + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { @@ -207,7 +210,12 @@ describe('user render timeout limit', function () { it('layergroup creation works but vector tile request fails due to render timeout', function (done) { const params = { format: 'mvt', - status: 400 + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { @@ -243,7 +251,12 @@ describe('user render timeout limit', function () { const params = { layers: 'mapnik', format: 'grid.json', - status: 400 + response: { + status: 400, + headers: { + 'Content-Type': 'application/x-protobuf' + } + } }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { From 435d902e45bf3146fa13d4662eb7a230267dcff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:10:14 +0200 Subject: [PATCH 281/402] Expose function to clean all database connections in the pool --- test/support/test_helper.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/support/test_helper.js b/test/support/test_helper.js index fc9fcb34..49c6c1ee 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -128,11 +128,10 @@ afterEach(function(done) { }); }); - -afterEach(function () { +function cleanPGPoolConnections () { // TODO: this method will be replaced by psql.end pg.end(); -}); +} function deleteRedisKeys(keysToDelete, callback) { @@ -196,6 +195,7 @@ module.exports = { checkSurrogateKey: checkSurrogateKey, checkCache: checkCache, rmdirRecursiveSync: rmdirRecursiveSync, - configureMetadata + configureMetadata, + cleanPGPoolConnections }; From aa6d01f151a5fb614ce8ad3980035f10c6a1eed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:12:33 +0200 Subject: [PATCH 282/402] Add timeout test for attributes --- test/support/test-client.js | 104 ++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 1d66d2b6..8b6ab3e8 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -791,6 +791,110 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { ); }; +TestClient.prototype.getAttributes = function(params, callback) { + var self = this; + + if (!Number.isFinite(params.featureId)) { + throw new Error('featureId param must be a number') + } + + if (!Number.isFinite(params.layer)) { + throw new Error('layer param must be a number') + } + + var url = '/api/v1/map'; + + if (this.apiKey) { + url += '?' + qs.stringify({ api_key: this.apiKey }); + } + + var layergroupid; + + if (params.layergroupid) { + layergroupid = params.layergroupid + } + + step( + function createLayergroup() { + var next = this; + + if (layergroupid) { + return next(null, layergroupid); + } + + assert.response(self.server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + var parsedBody = JSON.parse(res.body); + + return next(null, parsedBody.layergroupid); + } + ); + }, + function getAttributes(err, _layergroupid) { + assert.ifError(err); + + var next = this; + + layergroupid = _layergroupid; + + url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`; + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }; + + var expectedResponse = params.response || { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + assert.response(self.server, request, expectedResponse, function (res, err) { + assert.ifError(err); + + var attributes = JSON.parse(res.body); + + next(null, res, attributes); + }); + }, + function finish(err, res, attributes) { + if (layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(err, res, attributes); + } + ); +}; + TestClient.prototype.drain = function(callback) { helper.deleteRedisKeys(this.keysToDelete, callback); }; From 04da57fe0cfc7b566a0315edd1c514a9499751de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:14:32 +0200 Subject: [PATCH 283/402] Do not create layergroup if it is already provided --- test/support/test-client.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 8b6ab3e8..34d04200 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -471,6 +471,11 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } var layergroupId; + + if (params.layergroupid) { + layergroupId = params.layergroupid + } + step( function createTemplate () { var next = this; @@ -512,6 +517,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { function createLayergroup(err, templateId) { var next = this; + if (layergroupId) { + return next(null, layergroupId); + } + var data = templateId ? params.placeholders : self.mapConfig var path = templateId ? urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : From f079c24554bbfc8c5d10e525770d4c7d9867f944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:22:13 +0200 Subject: [PATCH 284/402] Use parsed body variable --- test/support/test-client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 34d04200..2f322770 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -707,7 +707,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { return callback(err); } - return callback(null, JSON.parse(res.body)); + return callback(null, parsedBody); } ); }; From 9b5482489e2e9261a6fe216d04b5aa1e21136b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:23:17 +0200 Subject: [PATCH 285/402] Fix content-type defaults --- test/support/test-client.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 2f322770..04a701b4 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -586,32 +586,30 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }; - var expectedResponse = { - status: params.status || 200, + var expectedResponse = Object.assign({}, { + status: 200, headers: { - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'image/png' } - }; + }, params.response); + var isPng = format.match(/png$/); if (isPng) { request.encoding = 'binary'; - expectedResponse.headers['Content-Type'] = 'image/png'; } var isMvt = format.match(/mvt$/); if (isMvt) { request.encoding = 'binary'; - if (expectedResponse.status === 200) { expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; - } else if (expectedResponse.status === 204) { - expectedResponse.headers['Content-Type'] = undefined; } } + var isGeojson = format.match(/geojson$/); if (isGeojson) { From ac7509b01a0029c3728871502fe90ec17bb8fdfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:24:42 +0200 Subject: [PATCH 286/402] Expose function to clean up database connections --- test/support/test-client.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 04a701b4..e35824d7 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -967,6 +967,9 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) const publicuser = global.environment.postgres.user; + // we need to guarantee all new connections have the new settings + helper.cleanPGPoolConnections() + const psql = new PSQL({ user: 'postgres', dbname: dbname, @@ -989,3 +992,8 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) callback ); }; + +TestClient.cleanDatabaseConnections = function () { + helper.cleanPGPoolConnections() +} + From 227c2b336bcfc817f0cfb3ace69ac373ffe4e1c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:25:34 +0200 Subject: [PATCH 287/402] Uncomment database timeout configuration --- test/support/test-client.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index e35824d7..750a9f69 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -980,9 +980,9 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) step( function configureTimeouts () { const timeoutSQLs = [ - // `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, `ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, - // `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` + `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` ]; const group = this.group(); From b1114fc6063fdda92e3a554b5ac0f54f176d9439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:26:45 +0200 Subject: [PATCH 288/402] Add timeout test for toque.json tiles --- .../acceptance/user-database-timeout-limit.js | 272 +++++++++++++++--- 1 file changed, 236 insertions(+), 36 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index c59b26df..da9183c7 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -5,7 +5,7 @@ const TestClient = require('../support/test-client'); const pointSleepSql = ` SELECT - pg_sleep(0.5), + pg_sleep(0.3), 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id, 2 val @@ -17,7 +17,8 @@ const createMapConfig = ({ sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', - countBy = 'cartodb_id' + countBy = 'cartodb_id', + attributes } = {}) => ({ version, layers: [{ @@ -27,7 +28,8 @@ const createMapConfig = ({ id: 'a0' }, cartocss, - cartocss_version + cartocss_version, + attributes } }], analyses: [ @@ -54,15 +56,15 @@ const createMapConfig = ({ }); describe('user database timeout limit', function () { - beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 300, done); - }); - - afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); - }); - describe('dataview', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + beforeEach(function () { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); @@ -95,38 +97,236 @@ describe('user database timeout limit', function () { }); }); - describe('torque', function () { - beforeEach(function () { - const mapconfig = createMapConfig({ - type: 'torque', - cartocss: TestClient.CARTOCSS.TORQUE + describe('torque:', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors_with_context: [{ + type: 'layer', + message: 'TorqueRenderer: canceling statement due to statement timeout', + layer: { id: 'torque-layer0', index: 0, type: 'torque' } + }] + }); + + done(); + }); }); - this.testClient = new TestClient(mapconfig, 1234); }); - afterEach(function (done) { - this.testClient.drain(done); - }); + describe('fetching "torque.json" tile', function () { + before(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; - it('layergroup creation fails due to statement timeout', function (done) { - const expectedResponse = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { - assert.deepEqual(timeoutError, { - errors: ["TorqueRenderer: canceling statement due to statement timeout"], - errors_with_context: [{ - type: "layer", - message: "TorqueRenderer: canceling statement due to statement timeout", - layer: { id: 'torque-layer0', index: 0, type: "torque" } - }] + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); }); - done(); + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'torque.json', + layers: [ 0 ], + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { + assert.ifError(err); + + assert.deepEqual(attributes, { + errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors_with_context: [{ + type: 'unknown', + message: 'TorqueRenderer: canceling statement due to statement timeout', + }] + }); + + done(); + }); + }); + }); + }); + }); + + describe('attributes:', function () { + describe('while validating in map instatiation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + attributes: { + id: 'cartodb_id', + columns: [ 'val' ] + } + }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'canceling statement due to statement timeout' ], + errors_with_context: [{ + type: 'layer', + message: 'canceling statement due to statement timeout', + layer: { + id: 'layer0', + index: 0, + type: 'mapnik' + } + }] + }); + + done(); + }); + }); + }); + + describe('fetching by feature id', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + attributes: { + id: 'cartodb_id', + columns: [ 'val' ] + } + }); + + this.testClient = new TestClient(mapconfig, 1234); + + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + featureId: 1, + layer: 0, + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getAttributes(params, (err, res, attributes) => { + assert.ifError(err); + + assert.deepEqual(attributes, { + errors: ['canceling statement due to statement timeout'], + errors_with_context: [{ + type: 'unknown', + message: 'canceling statement due to statement timeout' + }] + }); + + done(); + }); + }); }); }); }); From 49d2f513c6c609e692c86383f7bfe1be5e5eb8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:51:23 +0200 Subject: [PATCH 289/402] Fix typo --- lib/cartodb/controllers/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 30883e04..b8b73569 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -229,7 +229,7 @@ function stripConnectionInfo(message) { .replace(/is the server.*encountered/im, 'encountered'); } -var ERROR_INFO_TO_EXOSE = { +var ERROR_INFO_TO_EXPOSE = { message: true, layer: true, type: true, @@ -238,7 +238,7 @@ var ERROR_INFO_TO_EXOSE = { }; function shouldBeExposed (prop) { - return !!ERROR_INFO_TO_EXOSE[prop]; + return !!ERROR_INFO_TO_EXPOSE[prop]; } function errorMessage(err) { From bb599025354ebccc978af7bed78fce13742cdcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 18:52:09 +0200 Subject: [PATCH 290/402] Refactoring tests hooks --- test/acceptance/user-database-timeout-limit.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index da9183c7..35add015 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -58,20 +58,18 @@ const createMapConfig = ({ describe('user database timeout limit', function () { describe('dataview', function () { beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); - }); - - beforeEach(function () { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - }); - - afterEach(function (done) { - this.testClient.drain(done); + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('layergroup creation works but dataview request fails due to statement timeout', function (done) { From 5974413d5cead17efc5cdca3d6d0b77e4500d1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 19:08:29 +0200 Subject: [PATCH 291/402] Use 429 to indicate timeout errors --- lib/cartodb/controllers/base.js | 7 +++++++ test/acceptance/user-database-timeout-limit.js | 10 +++++----- test/acceptance/user-render-timeout-limit.js | 8 ++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index b8b73569..c5afdecb 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -301,5 +301,12 @@ function statusFromErrorMessage(errMsg) { statusCode = 404; } } + else if ( -1 !== errMsg.indexOf('Render timed out') ) { + statusCode = 429; // under discussion + } + else if ( -1 !== errMsg.indexOf('canceling statement due to statement timeout') ) { + statusCode = 429; // under discussion + } + return statusCode; } diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 35add015..53b69775 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -75,7 +75,7 @@ describe('user database timeout limit', function () { it('layergroup creation works but dataview request fails due to statement timeout', function (done) { const params = { response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -117,7 +117,7 @@ describe('user database timeout limit', function () { it('fails due to statement timeout', function (done) { const expectedResponse = { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -182,7 +182,7 @@ describe('user database timeout limit', function () { format: 'torque.json', layers: [ 0 ], response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -231,7 +231,7 @@ describe('user database timeout limit', function () { it('layergroup creation fails due to statement timeout', function (done) { const expectedResponse = { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -304,7 +304,7 @@ describe('user database timeout limit', function () { featureId: 1, layer: 0, response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index cd5b03a7..e579a132 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -85,7 +85,7 @@ describe('user render timeout limit', function () { it('layergroup creation fails due to statement timeout', function (done) { const expectedResponse = { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -174,7 +174,7 @@ describe('user render timeout limit', function () { it('layergroup creation works and render tile fails', function (done) { var params = { response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -211,7 +211,7 @@ describe('user render timeout limit', function () { const params = { format: 'mvt', response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -252,7 +252,7 @@ describe('user render timeout limit', function () { layers: 'mapnik', format: 'grid.json', response: { - status: 400, + status: 429, headers: { 'Content-Type': 'application/x-protobuf' } From 1fdc0621e780348c902e01a620eab3b24ed06a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 19:36:07 +0200 Subject: [PATCH 292/402] Categorize timeout errors --- lib/cartodb/controllers/base.js | 34 +++++++++++++++++++ .../acceptance/user-database-timeout-limit.js | 18 +++++++--- test/acceptance/user-render-timeout-limit.js | 27 ++++++++++++--- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index c5afdecb..cdfc84ff 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -195,6 +195,9 @@ BaseController.prototype.send = function(req, res, body, status, headers) { BaseController.prototype.sendError = function(req, res, err, label) { var allErrors = Array.isArray(err) ? err : [err]; + + allErrors = populateTimeoutErrors(allErrors) + label = label || 'UNKNOWN'; err = allErrors[0] || new Error(label); allErrors[0] = err; @@ -310,3 +313,34 @@ function statusFromErrorMessage(errMsg) { return statusCode; } + +function isRenderTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('Render timed out')) +} + +function isDatasourceTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')) +} + +function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); +} + +function populateTimeoutErrors (errors) { + return errors.map(function (error) { + if (isTimeoutError(error)) { + error.type = 'limit' + } + + if (isRenderTimeoutError(error)) { + error.subtype = 'render' + } + + if (isDatasourceTimeoutError(error)) { + + error.subtype = 'datasource' + } + + return error; + }) +} diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 53b69775..241b900c 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -87,7 +87,11 @@ describe('user database timeout limit', function () { assert.deepEqual(dataview, { errors: ['canceling statement due to statement timeout'], - errors_with_context: [{ type: 'unknown', message: 'canceling statement due to statement timeout' }] + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'canceling statement due to statement timeout' + }] }); done(); @@ -127,7 +131,8 @@ describe('user database timeout limit', function () { assert.deepEqual(timeoutError, { errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], errors_with_context: [{ - type: 'layer', + type: 'limit', + subtype: 'datasource', message: 'TorqueRenderer: canceling statement due to statement timeout', layer: { id: 'torque-layer0', index: 0, type: 'torque' } }] @@ -195,7 +200,8 @@ describe('user database timeout limit', function () { assert.deepEqual(attributes, { errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], errors_with_context: [{ - type: 'unknown', + type: 'limit', + subtype: 'datasource', message: 'TorqueRenderer: canceling statement due to statement timeout', }] }); @@ -241,7 +247,8 @@ describe('user database timeout limit', function () { assert.deepEqual(timeoutError, { errors: [ 'canceling statement due to statement timeout' ], errors_with_context: [{ - type: 'layer', + type: 'limit', + subtype: 'datasource', message: 'canceling statement due to statement timeout', layer: { id: 'layer0', @@ -317,7 +324,8 @@ describe('user database timeout limit', function () { assert.deepEqual(attributes, { errors: ['canceling statement due to statement timeout'], errors_with_context: [{ - type: 'unknown', + type: 'limit', + subtype: 'datasource', message: 'canceling statement due to statement timeout' }] }); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index e579a132..c5751bed 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -97,7 +97,8 @@ describe('user render timeout limit', function () { assert.deepEqual(timeoutError, { errors: ["Render timed out"], errors_with_context: [{ - type: "layer", + type: 'limit', + subtype: 'render', message: "Render timed out", layer: { id: "layer0", @@ -181,10 +182,18 @@ describe('user render timeout limit', function () { } }; - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { assert.ifError(err); - assert.equal(tile.errors[0], 'Render timed out'); + assert.deepEqual(timeoutError, { + errors: ["Render timed out"], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: "Render timed out" + }] + }); + done(); }); }); @@ -223,7 +232,11 @@ describe('user render timeout limit', function () { assert.deepEqual(tile, { errors: ['Render timed out'], - errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: 'Render timed out' + }] }); done(); @@ -264,7 +277,11 @@ describe('user render timeout limit', function () { assert.deepEqual(tile, { errors: ['Render timed out'], - errors_with_context: [{ type: 'unknown', message: 'Render timed out' }] + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: 'Render timed out' + }] }); done(); From 44fc34b1ce5c4449fb11266dd2292f10e5cabfb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 31 Jul 2017 20:07:31 +0200 Subject: [PATCH 293/402] Improve timeout error message --- lib/cartodb/controllers/base.js | 27 ++++++++----------- test/acceptance/limits.js | 12 ++++++--- .../acceptance/user-database-timeout-limit.js | 20 +++++++------- test/acceptance/user-render-timeout-limit.js | 16 +++++------ 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index cdfc84ff..5072d899 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -196,7 +196,7 @@ BaseController.prototype.send = function(req, res, body, status, headers) { BaseController.prototype.sendError = function(req, res, err, label) { var allErrors = Array.isArray(err) ? err : [err]; - allErrors = populateTimeoutErrors(allErrors) + allErrors = populateTimeoutErrors(allErrors); label = label || 'UNKNOWN'; err = allErrors[0] || new Error(label); @@ -304,22 +304,16 @@ function statusFromErrorMessage(errMsg) { statusCode = 404; } } - else if ( -1 !== errMsg.indexOf('Render timed out') ) { - statusCode = 429; // under discussion - } - else if ( -1 !== errMsg.indexOf('canceling statement due to statement timeout') ) { - statusCode = 429; // under discussion - } return statusCode; } function isRenderTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('Render timed out')) + return err.message && (-1 !== err.message.indexOf('Render timed out')); } function isDatasourceTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')) + return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); } function isTimeoutError (err) { @@ -328,19 +322,20 @@ function isTimeoutError (err) { function populateTimeoutErrors (errors) { return errors.map(function (error) { - if (isTimeoutError(error)) { - error.type = 'limit' - } - if (isRenderTimeoutError(error)) { - error.subtype = 'render' + error.subtype = 'render'; } if (isDatasourceTimeoutError(error)) { + error.subtype = 'datasource'; + } - error.subtype = 'datasource' + if (isTimeoutError(error)) { + error.message = 'You are over platform limits. Please contact us to know more details'; + error.type = 'limit'; + error.http_status = 429; } return error; - }) + }); } diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js index d0126623..f4156363 100644 --- a/test/acceptance/limits.js +++ b/test/acceptance/limits.js @@ -102,11 +102,13 @@ describe('render limits', function() { assert.response(server, createRequest(layergroup, user), { - status: 400 + status: 429 }, function(res) { var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, [ 'Render timed out' ]); + assert.deepEqual(parsed.errors, [ + 'You are over platform limits. Please contact us to know more details' + ]); done(); } ); @@ -167,11 +169,13 @@ describe('render limits', function() { encoding: 'binary' }, { - status: 400 + status: 429 }, function(res) { var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, ['Render timed out']); + assert.deepEqual(parsed.errors, [ + 'You are over platform limits. Please contact us to know more details' + ]); done(); } ); diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 241b900c..8662636f 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -86,11 +86,11 @@ describe('user database timeout limit', function () { assert.ifError(err); assert.deepEqual(dataview, { - errors: ['canceling statement due to statement timeout'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'canceling statement due to statement timeout' + message: 'You are over platform limits. Please contact us to know more details' }] }); @@ -129,11 +129,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors: [ 'You are over platform limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'TorqueRenderer: canceling statement due to statement timeout', + message: 'You are over platform limits. Please contact us to know more details', layer: { id: 'torque-layer0', index: 0, type: 'torque' } }] }); @@ -198,11 +198,11 @@ describe('user database timeout limit', function () { assert.ifError(err); assert.deepEqual(attributes, { - errors: [ 'TorqueRenderer: canceling statement due to statement timeout' ], + errors: [ 'You are over platform limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'TorqueRenderer: canceling statement due to statement timeout', + message: 'You are over platform limits. Please contact us to know more details', }] }); @@ -245,11 +245,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'canceling statement due to statement timeout' ], + errors: [ 'You are over platform limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'canceling statement due to statement timeout', + message: 'You are over platform limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, @@ -322,11 +322,11 @@ describe('user database timeout limit', function () { assert.ifError(err); assert.deepEqual(attributes, { - errors: ['canceling statement due to statement timeout'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'canceling statement due to statement timeout' + message: 'You are over platform limits. Please contact us to know more details' }] }); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index c5751bed..a986dd34 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -95,11 +95,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["Render timed out"], + errors: ["You are over platform limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "Render timed out", + message: "You are over platform limits. Please contact us to know more details", layer: { id: "layer0", index: 0, @@ -186,11 +186,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["Render timed out"], + errors: ["You are over platform limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "Render timed out" + message: "You are over platform limits. Please contact us to know more details" }] }); @@ -231,11 +231,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['Render timed out'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'Render timed out' + message: 'You are over platform limits. Please contact us to know more details' }] }); @@ -276,11 +276,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['Render timed out'], + errors: ['You are over platform limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'Render timed out' + message: 'You are over platform limits. Please contact us to know more details' }] }); From dd5209b9a724902142db0154502af62f320a7a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 09:39:37 +0200 Subject: [PATCH 294/402] Add torque.png timeout error test --- .../acceptance/user-database-timeout-limit.js | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 8662636f..6eb2f0e5 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -55,6 +55,15 @@ const createMapConfig = ({ } }); +const DATASOURCE_TIMEOUT_ERROR = { + errors: ['You are over platform limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details' + }] +}; + describe('user database timeout limit', function () { describe('dataview', function () { beforeEach(function (done) { @@ -85,21 +94,14 @@ describe('user database timeout limit', function () { this.testClient.getDataview('count', params, (err, dataview) => { assert.ifError(err); - assert.deepEqual(dataview, { - errors: ['You are over platform limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details' - }] - }); + assert.deepEqual(dataview, DATASOURCE_TIMEOUT_ERROR); done(); }); }); }); - describe('torque:', function () { + describe('torque', function () { describe('while validating in layergroup creation', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ @@ -143,8 +145,8 @@ describe('user database timeout limit', function () { }); }); - describe('fetching "torque.json" tile', function () { - before(function (done) { + describe('fetching torque tiles', function () { + beforeEach(function (done) { const mapconfig = createMapConfig({ type: 'torque', cartocss: TestClient.CARTOCSS.TORQUE @@ -181,7 +183,7 @@ describe('user database timeout limit', function () { TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); }); - it('fails due to statement timeout', function (done) { + it('"torque.json" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'torque.json', @@ -197,14 +199,29 @@ describe('user database timeout limit', function () { this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { assert.ifError(err); - assert.deepEqual(attributes, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', - }] - }); + assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + + it('".png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'torque.png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { + assert.ifError(err); + + assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); done(); }); @@ -321,14 +338,7 @@ describe('user database timeout limit', function () { this.testClient.getAttributes(params, (err, res, attributes) => { assert.ifError(err); - assert.deepEqual(attributes, { - errors: ['You are over platform limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details' - }] - }); + assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); done(); }); From 0c4e67d6a8ea9f4aebdd9898cf5114e45064ecf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 10:21:39 +0200 Subject: [PATCH 295/402] Implemented database timeout test while requesting tiles --- .../acceptance/user-database-timeout-limit.js | 328 +++++++++++++++++- 1 file changed, 321 insertions(+), 7 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 6eb2f0e5..62aae5ce 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -11,12 +11,21 @@ const pointSleepSql = ` 2 val `; +const validationPointSleepSql = ` + SELECT + pg_sleep(0.3), + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + const createMapConfig = ({ version = '1.6.0', type = 'cartodb', sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS, cartocss_version = '2.3.0', + interactivity = 'cartodb_id', countBy = 'cartodb_id', attributes } = {}) => ({ @@ -29,7 +38,8 @@ const createMapConfig = ({ }, cartocss, cartocss_version, - attributes + attributes, + interactivity } }], analyses: [ @@ -91,16 +101,320 @@ describe('user database timeout limit', function () { } }; - this.testClient.getDataview('count', params, (err, dataview) => { + this.testClient.getDataview('count', params, (err, timeoutError) => { assert.ifError(err); - assert.deepEqual(dataview, DATASOURCE_TIMEOUT_ERROR); + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); done(); }); }); }); + describe('raster', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching raster tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + + describe('vector', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching vector tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it('"mvt" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'mvt', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + + + describe('interactivity', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' }); + this.testClient = new TestClient(mapconfig, 1234); + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching interactivity tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ interactivity: 'val' }); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + }); + + afterEach(function (done) { + TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + }); + + it.skip('"grid.json" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'grid.json', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + describe('torque', function () { describe('while validating in layergroup creation', function () { beforeEach(function (done) { @@ -196,10 +510,10 @@ describe('user database timeout limit', function () { } }; - this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { assert.ifError(err); - assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); done(); }); @@ -335,10 +649,10 @@ describe('user database timeout limit', function () { } }; - this.testClient.getAttributes(params, (err, res, attributes) => { + this.testClient.getAttributes(params, (err, res, timeoutError) => { assert.ifError(err); - assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); done(); }); From 9af372381c0cdc3ce0693f7c99724eb17ab205a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 10:29:29 +0200 Subject: [PATCH 296/402] Fix content-type assertion --- test/acceptance/user-render-timeout-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index a986dd34..382a8035 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -267,7 +267,7 @@ describe('user render timeout limit', function () { response: { status: 429, headers: { - 'Content-Type': 'application/x-protobuf' + 'Content-Type': 'application/json; charset=utf-8' } } }; From aa4bb62f38153bdd0f91c423a557e49951a2c430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 10:29:46 +0200 Subject: [PATCH 297/402] Fix test --- test/acceptance/user-database-timeout-limit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 62aae5ce..1caeafec 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -390,11 +390,11 @@ describe('user database timeout limit', function () { TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); }); - it.skip('"grid.json" fails due to statement timeout', function (done) { + it('"grid.json" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'grid.json', - layers: [ 0 ], + layers: 'mapnik', response: { status: 429, headers: { From 91b3e373b756856785cb4516881218de41b6148c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 11:46:48 +0200 Subject: [PATCH 298/402] Add helper method to fetch static images --- test/support/test-client.js | 104 ++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/test/support/test-client.js b/test/support/test-client.js index 750a9f69..6eeb260d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -710,6 +710,110 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { ); }; +TestClient.prototype.getStaticCenter = function (params, callback) { + var self = this; + + let { layergroupid, z, lat, lng, width, height, format } = params + + var url = `/api/v1/map/`; + + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + step( + function createLayergroup() { + var next = this; + + if (layergroupid) { + return next(null, layergroupid); + } + + var data = self.mapConfig + var path = url; + + assert.response(self.server, + { + url: path, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(data) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).layergroupid); + } + ); + }, + function getStaticResult(err, _layergroupid) { + assert.ifError(err); + + var next = this; + + layergroupid = _layergroupid; + + url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}` + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + encoding: 'binary', + method: 'GET', + headers: { + host: 'localhost' + } + }; + + var expectedResponse = Object.assign({}, { + status: 200, + headers: { + 'Content-Type': 'image/png' + } + }, params.response); + + assert.response(self.server, request, expectedResponse, function(res, err) { + assert.ifError(err); + + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; + } + + next(null, res, body); + }); + }, + function finish(err, res, image) { + if (layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + return callback(err, res, image); + } + ); +}; + TestClient.prototype.getNodeStatus = function(nodeName, callback) { var self = this; From 40af73d524da922bef3c5b34d8fa86d2df8ab30a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 11:47:50 +0200 Subject: [PATCH 299/402] Implement test to check render timeout error for static api --- test/acceptance/user-render-timeout-limit.js | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 382a8035..23b3cc70 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -288,5 +288,105 @@ describe('user render timeout limit', function () { }); }); }); + + describe('static images', function () { + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but static image fails due to render timeout', function (done) { + const params = { + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png' + }; + + this.testClient.getStaticCenter(params, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + }); + + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works and render tile fails', function (done) { + const params = { + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, function (err, res, timeoutError) { + assert.ifError(err); + + assert.deepEqual(timeoutError, { + errors: ["You are over platform limits. Please contact us to know more details"], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: "You are over platform limits. Please contact us to know more details" + }] + }); + + done(); + }); + }); + }); + }); }); From 7d136031638cd93c2ede04fff753f0d3fa743d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 11:58:43 +0200 Subject: [PATCH 300/402] Implement test to validate database timeout error for static api --- .../acceptance/user-database-timeout-limit.js | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 1caeafec..a320d08f 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -208,6 +208,32 @@ describe('user database timeout limit', function () { done(); }); }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, function (err, res, timeoutError) { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); }); }); }); From 4c3e3005aa3cc51117577b7846990db3600f7317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:52:34 +0200 Subject: [PATCH 301/402] Apply asset fallback to database timeout errors --- lib/cartodb/server.js | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 13d64804..7968dc5f 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -118,8 +118,27 @@ module.exports = function(serverOptions) { var onTileErrorStrategy; if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { - if (err && err.message === 'Render timed out' && format === 'png') { - return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {}); + + function isRenderTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('Render timed out')); + } + + function isDatasourceTimeoutError (err) { + return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); + } + + function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); + } + + function isRasterFormat (format) { + return format === 'png' || format === 'jpg' + } + + if (isTimeoutError(err) && isRasterFormat(format)) { + return callback(null, timeoutErrorTile, { + 'Content-Type': 'image/png', + }, {}); } else { return callback(err, tile, headers, stats); } From 05ddf1d505b4ccee25bb3589be5fc84d9154ad96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:53:29 +0200 Subject: [PATCH 302/402] Add test to check if asset fallback is working when enabled and database timeout erro happens --- .../acceptance/user-database-timeout-limit.js | 193 +++++++++++++----- 1 file changed, 138 insertions(+), 55 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index a320d08f..4b9df8a4 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -3,6 +3,8 @@ require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; + const pointSleepSql = ` SELECT pg_sleep(0.3), @@ -153,31 +155,6 @@ describe('user database timeout limit', function () { }); describe('fetching raster tiles', function () { - beforeEach(function (done) { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - const expectedResponse = { - status: 200, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - this.testClient.getLayergroup(expectedResponse, (err, res) => { - if (err) { - return done(err); - } - - this.layergroupid = res.layergroupid; - - done(); - }); - }); - - afterEach(function (done) { - this.testClient.drain(done); - }); - describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); @@ -187,51 +164,157 @@ describe('user database timeout limit', function () { TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); }); - it('"png" fails due to statement timeout', function (done) { - const params = { - layergroupid: this.layergroupid, - format: 'png', - layers: [ 0 ], - response: { - status: 429, + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, headers: { 'Content-Type': 'application/json; charset=utf-8' } - } - }; + }; - this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { - assert.ifError(err); + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + this.layergroupid = res.layergroupid; - done(); + done(); + }); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.drain(done); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ] + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png' + }; + + this.testClient.getStaticCenter(params, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); }); }); - it('"static png" fails due to statement timeout', function (done) { - const params = { - layergroupid: this.layergroupid, - zoom: 0, - lat: 0, - lng: 0, - width: 256, - height: 256, - format: 'png', - response: { - status: 429, + describe('with onTileErrorStrategy DISABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, headers: { 'Content-Type': 'application/json; charset=utf-8' } - } - }; + }; - this.testClient.getStaticCenter(params, function (err, res, timeoutError) { - assert.ifError(err); + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + this.layergroupid = res.layergroupid; - done(); + done(); + }); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.drain(done); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); }); }); }); From 46fee774bdba8b9c54323e3e0d17e530c0dcaf56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:54:41 +0200 Subject: [PATCH 303/402] Fix misconfiguration in test's hook --- test/acceptance/user-render-timeout-limit.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 23b3cc70..2fab657f 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -303,6 +303,8 @@ describe('user render timeout limit', function () { }); afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { if (err) { return done(err); From 9258ad7eccb8663fbd58885e96536929b4de8258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 12:56:03 +0200 Subject: [PATCH 304/402] Fix style typo --- lib/cartodb/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7968dc5f..7b6fc33a 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -132,7 +132,7 @@ module.exports = function(serverOptions) { } function isRasterFormat (format) { - return format === 'png' || format === 'jpg' + return format === 'png' || format === 'jpg'; } if (isTimeoutError(err) && isRasterFormat(format)) { From 262f854e68dfe787525a5962ddc49a557fd6bb21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 13:10:55 +0200 Subject: [PATCH 305/402] Remove error wrapping --- lib/cartodb/models/dataview/base.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 7c3dbe6f..90d8a299 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -10,8 +10,7 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { this.sql(psql, override, function(err, query) { psql.query(query, function(err, result) { if (err) { - var error = new Error(err.message); - return callback(error, result); + return callback(err, result); } result = self.format(result, override); From 0594407b3837b651944c69ebf5b703f8092363e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 15:03:09 +0200 Subject: [PATCH 306/402] Change error message --- lib/cartodb/controllers/base.js | 2 +- test/acceptance/limits.js | 4 ++-- .../acceptance/user-database-timeout-limit.js | 24 +++++++++---------- test/acceptance/user-render-timeout-limit.js | 20 ++++++++-------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 5072d899..e3d2a407 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -331,7 +331,7 @@ function populateTimeoutErrors (errors) { } if (isTimeoutError(error)) { - error.message = 'You are over platform limits. Please contact us to know more details'; + error.message = 'You are over platform\'s limits. Please contact us to know more details'; error.type = 'limit'; error.http_status = 429; } diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js index f4156363..372fb189 100644 --- a/test/acceptance/limits.js +++ b/test/acceptance/limits.js @@ -107,7 +107,7 @@ describe('render limits', function() { function(res) { var parsed = JSON.parse(res.body); assert.deepEqual(parsed.errors, [ - 'You are over platform limits. Please contact us to know more details' + 'You are over platform\'s limits. Please contact us to know more details' ]); done(); } @@ -174,7 +174,7 @@ describe('render limits', function() { function(res) { var parsed = JSON.parse(res.body); assert.deepEqual(parsed.errors, [ - 'You are over platform limits. Please contact us to know more details' + 'You are over platform\'s limits. Please contact us to know more details' ]); done(); } diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 4b9df8a4..e1ff7bda 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -68,11 +68,11 @@ const createMapConfig = ({ }); const DATASOURCE_TIMEOUT_ERROR = { - errors: ['You are over platform limits. Please contact us to know more details'], + errors: ['You are over platform\'s limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details' + message: 'You are over platform\'s limits. Please contact us to know more details' }] }; @@ -140,11 +140,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -348,11 +348,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -450,11 +450,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -554,11 +554,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'torque-layer0', index: 0, type: 'torque' } }] }); @@ -685,11 +685,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform limits. Please contact us to know more details' ], + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform limits. Please contact us to know more details', + message: 'You are over platform\'s limits. Please contact us to know more details', layer: { id: 'layer0', index: 0, diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 2fab657f..45e41f89 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -95,11 +95,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform limits. Please contact us to know more details"], + errors: ["You are over platform\'s limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform limits. Please contact us to know more details", + message: "You are over platform\'s limits. Please contact us to know more details", layer: { id: "layer0", index: 0, @@ -186,11 +186,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform limits. Please contact us to know more details"], + errors: ["You are over platform\'s limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform limits. Please contact us to know more details" + message: "You are over platform\'s limits. Please contact us to know more details" }] }); @@ -231,11 +231,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['You are over platform limits. Please contact us to know more details'], + errors: ['You are over platform\'s limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'You are over platform limits. Please contact us to know more details' + message: 'You are over platform\'s limits. Please contact us to know more details' }] }); @@ -276,11 +276,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(tile, { - errors: ['You are over platform limits. Please contact us to know more details'], + errors: ['You are over platform\'s limits. Please contact us to know more details'], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'You are over platform limits. Please contact us to know more details' + message: 'You are over platform\'s limits. Please contact us to know more details' }] }); @@ -377,11 +377,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform limits. Please contact us to know more details"], + errors: ["You are over platform\'s limits. Please contact us to know more details"], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform limits. Please contact us to know more details" + message: "You are over platform\'s limits. Please contact us to know more details" }] }); From 90345082441fc88e6cad11c52672430638ab3320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 17:15:45 +0200 Subject: [PATCH 307/402] Support automattic aggregation only when aggregation para is set to 'auto' --- lib/cartodb/backends/dataview.js | 1 + lib/cartodb/models/dataview/histogram.js | 10 ++++++++-- test/acceptance/dataviews/histogram.js | 6 ++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 5fe1745a..0c9a30fe 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -21,6 +21,7 @@ function DataviewBackend(analysisBackend) { } var DATE_AGGREGATIONS = { + 'auto': true, 'minute': true, 'hour': true, 'day': true, diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7f2d13d7..436a0368 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -5,6 +5,8 @@ var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); dot.templateSettings.strip = false; +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); + var dateIntervalQueryTpl = dot.template([ 'WITH', 'dates AS (', @@ -331,10 +333,14 @@ Histogram.prototype._buildQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; - if (this._columnType === 'date') { + if (this._columnType === 'date' && this.aggregation !== undefined) { return this._buildDateHistogramQuery(psql, override, callback); } + if (this._columnType === 'date') { + _column = columnCastTpl({column: _column}); + } + filteredQuery = filteredQueryTpl({ _isFloatColumn: this._columnType === 'float', _query: _query, @@ -434,7 +440,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; - if (!_aggregation) { + if (_aggregation === 'auto') { this.getAutomaticAggregation(psql, function (err, aggregation) { if (err || aggregation === 'none') { this.aggregation = 'day'; diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 58084aa4..4d48ab33 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -160,7 +160,8 @@ describe('histogram-dataview for date column type', function() { }, type: 'histogram', options: { - column: 'd' + column: 'd', + aggregation: 'auto' } }, date_histogram: { @@ -179,7 +180,8 @@ describe('histogram-dataview for date column type', function() { }, type: 'histogram', options: { - column: 'd' + column: 'd', + aggregation: 'auto' } }, minute_histogram: { From ab879e263465e024af45b235952d2c2760a902b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 19:13:55 +0200 Subject: [PATCH 308/402] Use new version of `getUserTimeoutRenderLimits` --- lib/cartodb/api/user_limits_api.js | 17 ++++++++++------- yarn.lock | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 7e004e31..8954c261 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -87,13 +87,16 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call return next(err); } - self.metadataBackend.getUserTimeoutRenderLimits(username, authorized, next); + self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { + if (err) { + return next(err) + } + + next(null, { + render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic + }); + }); }, - function setTilerRenderLimit(err, timeoutRenderLimit) { - if (err) { - return callback(err); - } - return callback(null, timeoutRenderLimit); - } + callback ); }; diff --git a/yarn.lock b/yarn.lock index d7c06ac0..63d9f6fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -260,7 +260,7 @@ cartodb-query-tables@0.2.0: cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: version "0.13.3" - resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/0c0e5c410243bd4a16e9d94b52821e9d6533eb16" + resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/530ae2736a184a3b9c9e978efa7dcb4faae7abee" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" From 3574700c2d4c13cecffc438105ecaa645833067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Aug 2017 11:07:44 +0200 Subject: [PATCH 309/402] Remove tiler render limit --- lib/cartodb/api/user_limits_api.js | 21 +- test/acceptance/limits.js | 315 ----------------------------- yarn.lock | 2 +- 3 files changed, 2 insertions(+), 336 deletions(-) delete mode 100644 test/acceptance/limits.js diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 8954c261..74d0cf21 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -24,21 +24,6 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) }; step( - function getTilerLimit() { - var next = this; - self.getTilerRenderLimit(username, function (err, tilerRenderLimit) { - if (err) { - return callback(err); - } - - if (Number.isFinite(tilerRenderLimit)) { - limits.render = tilerRenderLimit; - return callback(null, limits); - } - - return next(); - }); - }, function getTimeoutLimit() { self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { if (err) { @@ -57,10 +42,6 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) ); }; -UserLimitsApi.prototype.getTilerRenderLimit = function (username, callback) { - this.metadataBackend.getTilerRenderLimit(username, callback); -}; - UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { var self = this; @@ -89,7 +70,7 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { if (err) { - return next(err) + return next(err); } next(null, { diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js deleted file mode 100644 index 372fb189..00000000 --- a/test/acceptance/limits.js +++ /dev/null @@ -1,315 +0,0 @@ -var testHelper = require('../support/test_helper'); - -var assert = require('../support/assert'); -var _ = require('underscore'); -var redis = require('redis'); - -var CartodbWindshaft = require('../../lib/cartodb/server'); -var serverOptions = require('../../lib/cartodb/server_options'); - -var LayergroupToken = require('../support/layergroup-token'); - -describe('render limits', function() { - - var layergroupUrl = '/api/v1/map'; - - var redisClient = redis.createClient(global.environment.redis.port); - - var server; - var keysToDelete; - beforeEach(function() { - keysToDelete = {}; - server = new CartodbWindshaft(serverOptions); - server.setMaxListeners(0); - }); - - afterEach(function(done) { - testHelper.deleteRedisKeys(keysToDelete, done); - }); - - var user = 'localhost'; - - var pointSleepSql = "SELECT pg_sleep(0.5)," + - " 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id"; - var pointCartoCss = '#layer { marker-fill:red; }'; - var polygonSleepSql = "SELECT pg_sleep(0.5)," + - " ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id"; - var polygonCartoCss = '#layer { polygon-fill:red; }'; - - function singleLayergroupConfig(sql, cartocss) { - return { - version: '1.0.0', - layers: [ - { - type: 'mapnik', - options: { - sql: sql, - cartocss: cartocss, - cartocss_version: '2.0.1' - } - } - ] - }; - } - - function createRequest(layergroup, userHost) { - return { - url: layergroupUrl, - method: 'POST', - headers: { - host: userHost, - 'Content-Type': 'application/json' - }, - data: JSON.stringify(layergroup) - }; - } - - function withRenderLimit(user, renderLimit, callback) { - redisClient.SELECT(5, function(err) { - if (err) { - return callback(err); - } - var userLimitsKey = 'limits:tiler:' + user; - redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) { - if (err) { - return callback(err); - } - keysToDelete[userLimitsKey] = 5; - return callback(); - }); - }); - - } - - describe('with onTileErrorStrategy DISABLED', function() { - var onTileErrorStrategyEnabled; - before(function() { - onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy; - global.environment.enabledFeatures.onTileErrorStrategy = false; - }); - - after(function() { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled; - }); - - it("layergroup creation fails if test tile is slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 429 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, [ - 'You are over platform\'s limits. Please contact us to know more details' - ]); - done(); - } - ); - }); - }); - - it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) { - withRenderLimit(user, 5000, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.ok(parsed.layergroupid); - keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - done(); - } - ); - }); - }); - - - it("layergroup creation works if test tile is fast but tile request fails if they are slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 429 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, [ - 'You are over platform\'s limits. Please contact us to know more details' - ]); - done(); - } - ); - - } - ); - }); - }); - - it("tile request does not fail if user limit is high enough", function(done) { - withRenderLimit(user, 5000, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 200, - headers: { - 'Content-Type': 'image/png' - } - }, - function(res, err) { - done(err); - } - ); - - } - ); - }); - }); - - }); - - describe('with onTileErrorStrategy', function() { - - it("layergroup creation works even if test tile is slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.ok(parsed.layergroupid); - keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - done(); - } - ); - }); - }); - - it("layergroup creation and tile requests works even if they are slow but returns fallback", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 200, - headers: { - 'Content-Type': 'image/png' - } - }, - function(res, err) { - if (err) { - done(err); - } - var referenceImagePath = './test/fixtures/render-timeout-fallback.png'; - assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25, - function(imgErr/*, similarity*/) { - done(imgErr); - } - ); - } - ); - - } - ); - }); - }); - - }); - -}); diff --git a/yarn.lock b/yarn.lock index 63d9f6fe..ebc2c0d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -260,7 +260,7 @@ cartodb-query-tables@0.2.0: cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: version "0.13.3" - resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/530ae2736a184a3b9c9e978efa7dcb4faae7abee" + resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/4c0586009dbfd464eb9e742bed39170334f40453" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" From 0a57e86cb8d34a5e48e5e1a690e2fa06d58ad976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Aug 2017 12:06:10 +0200 Subject: [PATCH 310/402] Do not build data histogram infinitely when overriding aggregation with auto mode --- lib/cartodb/models/dataview/histogram.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 436a0368..4c01cc46 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -447,6 +447,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac } else { this.aggregation = aggregation; } + override.aggregation = this.aggregation; this._buildDateHistogramQuery(psql, override, callback); }.bind(this)); return null; From 1bc5c044890cfe2ed169a93f7887275017cfa33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Aug 2017 13:15:40 +0200 Subject: [PATCH 311/402] Remove unused method --- test/support/test-client.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 6eeb260d..15448f68 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1097,7 +1097,4 @@ TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) ); }; -TestClient.cleanDatabaseConnections = function () { - helper.cleanPGPoolConnections() -} From b4799124e65c34b8a43af3f6722715bba9bf874f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 2 Aug 2017 17:28:46 +0200 Subject: [PATCH 312/402] Exclude non-finite values when computing ramps --- lib/cartodb/backends/turbo-carto-postgres-datasource.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/turbo-carto-postgres-datasource.js b/lib/cartodb/backends/turbo-carto-postgres-datasource.js index 8c0a93c7..65109325 100644 --- a/lib/cartodb/backends/turbo-carto-postgres-datasource.js +++ b/lib/cartodb/backends/turbo-carto-postgres-datasource.js @@ -10,7 +10,13 @@ function createTemplate(method) { 'max({{=it._column}}) max_val,', 'avg({{=it._column}}) avg_val,', method, - 'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL' + 'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL', + 'AND', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')); } From 7fed91900d952c18ee0588dacbc1c92a8eb8aec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 12:19:34 +0200 Subject: [PATCH 313/402] Handle error --- lib/cartodb/models/dataview/base.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index f8eb1027..280487b4 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -8,6 +8,10 @@ module.exports = BaseDataview; BaseDataview.prototype.getResult = function(psql, override, callback) { var self = this; this.sql(psql, override, function(err, query) { + if (err) { + return callback(err); + } + psql.query(query, function(err, result) { if (err) { From 90ded34af78df9c763836530aec637fd0e3c74dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 12:22:30 +0200 Subject: [PATCH 314/402] Do not fail if layergroup is undefined --- test/support/test-client.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 899c239f..3b9a003d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -404,8 +404,11 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }, function finish(err, dataview) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + return callback(err, dataview); } ); From b56d2ec30b584c0751ac469b700426524c5a7b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 12:24:05 +0200 Subject: [PATCH 315/402] Validate aggregation value --- lib/cartodb/backends/dataview.js | 14 +-- lib/cartodb/models/dataview/histogram.js | 17 ++++ test/acceptance/dataviews/histogram.js | 121 +++++++++++++++++++++++ yarn.lock | 12 +-- 4 files changed, 146 insertions(+), 18 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 0c9a30fe..29dcd903 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,17 +20,6 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } -var DATE_AGGREGATIONS = { - 'auto': true, - 'minute': true, - 'hour': true, - 'day': true, - 'week': true, - 'month': true, - 'quarter': true, - 'year': true -}; - module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { @@ -116,7 +105,8 @@ function getOverrideParams(params, ownFilter) { {ownFilter: ownFilter} ); - if (params.aggregation && DATE_AGGREGATIONS.hasOwnProperty(params.aggregation)) { + // validation will be delegated to the proper dataview + if (params.aggregation !== undefined) { overrideParams.aggregation = params.aggregation; } diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4c01cc46..13a9d998 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -434,12 +434,29 @@ Histogram.prototype._shouldOverrideBins = function (override) { return override && _.has(override, 'bins'); }; +var DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + if (_aggregation === 'auto') { this.getAutomaticAggregation(psql, function (err, aggregation) { if (err || aggregation === 'none') { diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 4d48ab33..51808d67 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -766,3 +766,124 @@ describe('histogram-dataview: special float valuer', function() { }); }); }); + +describe('histogram-dates: aggregation input value', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'day' + } + }, + bad_agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'wadus' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'2007-02-15 01:00:00\'::timestamp,', + '\'2008-04-09 01:00:00\'::timestamp,', + ' \'1 day\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should fail when aggregation values is not valid while instantiating the map', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + response: { + status: 400 + } + }; + + this.testClient.getDataview('bad_agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); + + it('should fail when aggregation values is not valid while fetching dataview result', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + aggregation: 'wadus', + response: { + status: 400 + } + }; + + this.testClient.getDataview('agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index a835b0e0..49cbc74e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1288,7 +1288,7 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6, moment@^2.18.1: +moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -2103,7 +2103,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb3: version "2.3.1-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23" dependencies: @@ -2111,7 +2111,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: From 431ca9c56ff9ab0470137ee6cb1b599f83df686f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 15:15:37 +0200 Subject: [PATCH 316/402] Update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index a3228af1..cb84fe39 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.9.9 Released 2017-mm-dd + - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698 + ## 3.9.8 Released 2017-07-21 From fe7a2451ef9ba19fdd9eebfedc6b601ab892a6f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 15:25:47 +0200 Subject: [PATCH 317/402] Release 3.10.0 --- NEWS.md | 8 +++++--- package.json | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index cb84fe39..ed1ae8c5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,11 @@ # Changelog -## 3.9.9 -Released 2017-mm-dd +## 3.10.0 +Released 2017-08-03 - - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698 +Announcements: + - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698. + - Support special numeric values (±Infinity, NaN) for json responses #706 ## 3.9.8 diff --git a/package.json b/package.json index d6e95c12..66540af6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.9", + "version": "3.10.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 1d1a04643929316f7e7ea8dc197be7c0a7a02d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 15:33:32 +0200 Subject: [PATCH 318/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ed1ae8c5..cafe337e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.10.1 +Released 2017-mm-dd + + ## 3.10.0 Released 2017-08-03 diff --git a/package.json b/package.json index 66540af6..3763b4c5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.10.0", + "version": "3.10.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 6c063095a365d80478685ab64130017a49b00cb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 18:18:35 +0200 Subject: [PATCH 319/402] Going red: aggregation is undefined when automattic mode is enabled and timestamp start is 1970-01-01 (epoch) --- test/acceptance/dataviews/histogram.js | 72 ++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 51808d67..1e3a2d40 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -887,3 +887,75 @@ describe('histogram-dates: aggregation input value', function() { }); }); }); + + +describe('histogram-dates: timestamp starts at epoch', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + epoch_start_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-04 10:00:00\'::timestamp,', + '\'1984-01-04 10:00:00\'::timestamp,', + ' \'1 month\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should work when timestamp_start is epoch (1970-01-01 = 0)', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = {}; + + this.testClient.getDataview('epoch_start_histogram', override, function(err, dataview) { + assert.ifError(err); + + console.log(dataview); + const { aggregation, timestamp_start } = dataview; + + assert.equal(timestamp_start, 0); + assert.equal(aggregation, 'quarter'); + + done(); + }); + }); +}); From 48ad7059e1de5da9095bb7f0b69d8d0199c2e50d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 18:23:55 +0200 Subject: [PATCH 320/402] Going green: do not rely on falsy conditional --- lib/cartodb/models/dataview/histogram.js | 2 +- test/acceptance/dataviews/histogram.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 13a9d998..76b42c03 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -589,7 +589,7 @@ Histogram.prototype.format = function(result, override) { nans = firstRow.nans_count; binsStart = populateBinStart(override, firstRow); - if (timestampStart) { + if (Number.isFinite(timestampStart)) { aggregation = getAggregation(override, this.aggregation); offset = getOffset(override, this.offset); } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 1e3a2d40..a75cf2b1 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -949,11 +949,10 @@ describe('histogram-dates: timestamp starts at epoch', function() { this.testClient.getDataview('epoch_start_histogram', override, function(err, dataview) { assert.ifError(err); - console.log(dataview); const { aggregation, timestamp_start } = dataview; assert.equal(timestamp_start, 0); - assert.equal(aggregation, 'quarter'); + assert.equal(aggregation, 'month'); done(); }); From eed33fc76da3035d0441de5df906bc8d5cddc98b Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 3 Aug 2017 19:07:02 +0200 Subject: [PATCH 321/402] Add tests for excluding NaNs, Ininities from ramps --- .../turbo-carto-postgres-datasource.js | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/unit/cartodb/backends/turbo-carto-postgres-datasource.js diff --git a/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js new file mode 100644 index 00000000..8276bf25 --- /dev/null +++ b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js @@ -0,0 +1,46 @@ +var PostgresDatasource = require('../../../../lib/cartodb/backends/turbo-carto-postgres-datasource'); +var PSQL = require('cartodb-psql'); +var _ = require('underscore'); +var assert = require('assert'); + +describe('turbo-carto-postgres-datasource', function() { + + beforeEach(function () { + const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; + const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) + const psql = new PSQL({ + user: 'postgres', + dbname: dbname, + host: global.environment.postgres.host, + port: global.environment.postgres.port + }); + const sql = [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS values', + 'FROM generate_series(1, 1000) x' + ].join('\n') + this.datasource = new PostgresDatasource(psql, sql); + }); + + it('should ignore NaNs and Infinities when computing ramps', function(done) { + column = 'values'; + buckets = 4; + method = 'equal'; + this.datasource.getRamp(column, buckets, method, function(err, result) { + expected_result = { + ramp: [ 252, 501, 750, 999 ], + stats: { min_val: 3, max_val: 999, avg_val: 501 }, + strategy: undefined + }; + assert.deepEqual(result, expected_result); + done(); + }); + }); +}); From e01730e8e46d2d479e3dca82f0c640963fd0afc4 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 3 Aug 2017 19:16:08 +0200 Subject: [PATCH 322/402] Syntax fixes --- .../backends/turbo-carto-postgres-datasource.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js index 8276bf25..7e774023 100644 --- a/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js +++ b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js @@ -7,8 +7,6 @@ describe('turbo-carto-postgres-datasource', function() { beforeEach(function () { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; - const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) - const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) const psql = new PSQL({ user: 'postgres', dbname: dbname, @@ -25,16 +23,16 @@ describe('turbo-carto-postgres-datasource', function() { ' ELSE x', ' END AS values', 'FROM generate_series(1, 1000) x' - ].join('\n') + ].join('\n'); this.datasource = new PostgresDatasource(psql, sql); }); it('should ignore NaNs and Infinities when computing ramps', function(done) { - column = 'values'; - buckets = 4; - method = 'equal'; + var column = 'values'; + var buckets = 4; + var method = 'equal'; this.datasource.getRamp(column, buckets, method, function(err, result) { - expected_result = { + var expected_result = { ramp: [ 252, 501, 750, 999 ], stats: { min_val: 3, max_val: 999, avg_val: 501 }, strategy: undefined From c39a6a680626eac514964d381c6908f4341b0a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 4 Aug 2017 09:42:21 +0200 Subject: [PATCH 323/402] Update news --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index cafe337e..c97b3d73 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,10 @@ ## 3.10.1 Released 2017-mm-dd +Bug fixes: + - Exclude Infinities & NaNs from ramps #719. + - Fixed issue in time-series when aggregation starts at 1970-01-01 (epoch) #720. + ## 3.10.0 Released 2017-08-03 From e52cf0960ca8ac2e78da8db9984ccbc8f139cbb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 4 Aug 2017 09:46:00 +0200 Subject: [PATCH 324/402] Release 3.10.1 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index c97b3d73..cd1fc224 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.10.1 -Released 2017-mm-dd +Released 2017-08-04 Bug fixes: - Exclude Infinities & NaNs from ramps #719. From 7eae2a06180539d51f8dde9cc17bc0b2ebb62e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 4 Aug 2017 09:51:16 +0200 Subject: [PATCH 325/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index cd1fc224..9ca6d4a4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.10.2 +Released 2017-mm-dd + + ## 3.10.1 Released 2017-08-04 diff --git a/package.json b/package.json index 3763b4c5..df934d7a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.10.1", + "version": "3.10.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 399561d076a69ce5ba7aa9b21a147ccc9931d12e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 4 Aug 2017 17:30:46 +0200 Subject: [PATCH 326/402] Implement LZMA query param inflating as middleware The req2params method is doing too many things, this is an initial step to do fewer things in that method. --- lib/cartodb/controllers/base.js | 34 +--------------- lib/cartodb/middleware/lzma.js | 30 ++++++++++++++ lib/cartodb/server.js | 4 ++ test/unit/cartodb/lzmaMiddleware.test.js | 36 +++++++++++++++++ test/unit/cartodb/req2params.test.js | 51 +++++++++++------------- 5 files changed, 95 insertions(+), 60 deletions(-) create mode 100644 lib/cartodb/middleware/lzma.js create mode 100644 test/unit/cartodb/lzmaMiddleware.test.js diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 5aab7cb3..43235c82 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -4,9 +4,6 @@ var _ = require('underscore'); var step = require('step'); var debug = require('debug')('windshaft:cartodb'); -var LZMA = require('lzma').LZMA; -var lzmaWorker = new LZMA(); - // Whitelist query parameters and attach format var REQUEST_QUERY_PARAMS_WHITELIST = [ 'config', @@ -28,7 +25,7 @@ function BaseController(authApi, pgConnection) { module.exports = BaseController; -// jshint maxcomplexity:10 +// jshint maxcomplexity:8 /** * Whitelist input and get database name & default geometry type from * subdomain/user metadata held in CartoDB Redis @@ -38,35 +35,6 @@ module.exports = BaseController; BaseController.prototype.req2params = function(req, callback){ var self = this; - if ( req.query.lzma ) { - - // Decode (from base64) - var lzma = new Buffer(req.query.lzma, 'base64') - .toString('binary') - .split('') - .map(function(c) { - return c.charCodeAt(0) - 128; - }); - - - // Decompress - lzmaWorker.decompress( - lzma, - function(result) { - req.profiler.done('lzma'); - try { - delete req.query.lzma; - _.extend(req.query, JSON.parse(result)); - self.req2params(req, callback); - } catch (err) { - req.profiler.done('req2params'); - callback(new Error('Error parsing lzma as JSON: ' + err)); - } - } - ); - return; - } - var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST; if (Array.isArray(req.context.allowedQueryParams)) { allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams); diff --git a/lib/cartodb/middleware/lzma.js b/lib/cartodb/middleware/lzma.js new file mode 100644 index 00000000..d58f16cc --- /dev/null +++ b/lib/cartodb/middleware/lzma.js @@ -0,0 +1,30 @@ +'use strict'; + +var LZMA = require('lzma').LZMA; + +var lzmaWorker = new LZMA(); + +module.exports = function lzmaMiddleware(req, res, next) { + if (!req.query.hasOwnProperty('lzma')) { + return next(); + } + + // Decode (from base64) + var lzma = new Buffer(req.query.lzma, 'base64') + .toString('binary') + .split('') + .map(function(c) { + return c.charCodeAt(0) - 128; + }); + + // Decompress + lzmaWorker.decompress(lzma, function(result) { + try { + delete req.query.lzma; + Object.assign(req.query, JSON.parse(result)); + next(); + } catch (err) { + next(new Error('Error parsing lzma as JSON: ' + err)); + } + }); +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index deb88e16..7177b12f 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -4,6 +4,8 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var _ = require('underscore'); +var lzmaMiddleware = require('./middleware/lzma'); + var controller = require('./controllers'); var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); @@ -345,6 +347,8 @@ function bootstrap(opts) { next(); }); + app.use(lzmaMiddleware); + // temporary measure until we upgrade to newer version expressjs so we can check err.status app.use(function(err, req, res, next) { if (err) { diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js new file mode 100644 index 00000000..9a41030a --- /dev/null +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -0,0 +1,36 @@ +var assert = require('assert'); +var testHelper = require('../../support/test_helper'); + +var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma'); + +describe('lzma-middleware', function() { + + it('it should extend params with decoded lzma', function(done) { + var qo = { + config: { + version: '1.3.0' + } + }; + testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { + var req = { + headers: { + host:'localhost' + }, + query: { + api_key: 'test', + lzma: data + } + }; + lzmaMiddleware(req, {}, function(err) { + if ( err ) { + return done(err); + } + var query = req.query; + assert.deepEqual(qo.config, query.config); + assert.equal('test', query.api_key); + done(); + }); + }); + }); + +}); diff --git a/test/unit/cartodb/req2params.test.js b/test/unit/cartodb/req2params.test.js index 6293199e..7b055e99 100644 --- a/test/unit/cartodb/req2params.test.js +++ b/test/unit/cartodb/req2params.test.js @@ -1,6 +1,6 @@ var assert = require('assert'); var _ = require('underscore'); -var test_helper = require('../../support/test_helper'); +require('../../support/test_helper'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); @@ -98,34 +98,31 @@ describe('req2params', function() { }); }); - it('it should extend params with decoded lzma', function(done) { - var qo = { - config: { - version: '1.3.0' + it('it should remove invalid params', function(done) { + var config = { + version: '1.3.0' + }; + var req = { + headers: { + host:'localhost' + }, + query: { + non_included: 'toberemoved', + api_key: 'test', + style: 'override', + config: config } }; - test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { - var req = { - headers: { - host:'localhost' - }, - query: { - non_included: 'toberemoved', - api_key: 'test', - style: 'override', - lzma: data - } - }; - baseController.req2params(prepareRequest(req), function(err, req) { - if ( err ) { - return done(err); - } - var query = req.params; - assert.deepEqual(qo.config, query.config); - assert.equal('test', query.api_key); - assert.equal(undefined, query.non_included); - done(); - }); + baseController.req2params(prepareRequest(req), function(err, req) { + if (err) { + return done(err); + } + var query = req.params; + assert.deepEqual(config, query.config); + assert.equal('test', query.api_key); + assert.equal(undefined, query.non_included); + assert.equal(undefined, query.style); + done(); }); }); From 660078f284caf65af724209f7c80b8b5dc48bfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 16:53:08 +0200 Subject: [PATCH 327/402] Fix minor issues with timezones --- NEWS.md | 6 + lib/cartodb/models/dataview/histogram.js | 4 +- test/acceptance/dataviews/histogram.js | 185 ++++++++++++++++++++++- 3 files changed, 192 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9ca6d4a4..3d5d5722 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,12 @@ ## 3.10.2 Released 2017-mm-dd +Announcements: + - Allow to override with any aggregation for histograms instantiated w/o aggregation. + +Bug fixes: + - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + ## 3.10.1 Released 2017-08-04 diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 76b42c03..891cc2ec 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -246,7 +246,7 @@ var dateHistogramQueryTpl = dot.template([ ' date_part(', ' \'epoch\', ', ' date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', @@ -333,7 +333,7 @@ Histogram.prototype._buildQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; - if (this._columnType === 'date' && this.aggregation !== undefined) { + if (this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined)) { return this._buildDateHistogramQuery(psql, override, callback); } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index a75cf2b1..84e2c15e 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -888,7 +888,6 @@ describe('histogram-dates: aggregation input value', function() { }); }); - describe('histogram-dates: timestamp starts at epoch', function() { afterEach(function(done) { @@ -958,3 +957,187 @@ describe('histogram-dates: timestamp starts at epoch', function() { }); }); }); + +describe('histogram-dates: trunc timestamp for each bin respecting user\'s timezone', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should return histogram with two buckets', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); +}); + + +describe('histogram: be able to override with aggregation for histograms instantiated w/o aggregation', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should apply aggregation to the histogram', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); +}); From 1f8da14c2ae6dff1e3353d04ac821f89cfe36430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 18:27:24 +0200 Subject: [PATCH 328/402] Cast to timestamp start_date and end_date to calculate bins when date column is timestamptz --- lib/cartodb/models/dataview/histogram.js | 4 +- test/acceptance/dataviews/histogram.js | 88 ++++++++++++++++-------- 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 891cc2ec..a044bfc7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -180,9 +180,9 @@ var dateBasicsQueryTpl = dot.template([ ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._offset}}\'', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )) AS start_date,', - ' max({{=it._column}} AT TIME ZONE \'{{=it._offset}}\') AS end_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 84e2c15e..3baf002b 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -991,6 +991,16 @@ describe('histogram-dates: trunc timestamp for each bin respecting user\'s timez column: 'd', aggregation: 'auto' } + }, + timezone_epoch_histogram_tz: { + source: { + id: 'a1' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } } }, [ @@ -1007,45 +1017,69 @@ describe('histogram-dates: trunc timestamp for each bin respecting user\'s timez ') date' ].join(' ') } + }, + { + id: 'a1', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamptz,', + '\'1970-01-01 01:59:00\'::timestamptz,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } } ] ); - it('should return histogram with two buckets', function(done) { - this.testClient = new TestClient(mapConfig, 1234); + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'timezone_epoch_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'timezone_epoch_histogram' + }]; - const override = { - aggregation: 'day', - offset: '-3600' - }; + dateHistogramsUseCases.forEach(function (test) { + it('should return histogram with two buckets ' + test.desc , function(done) { + this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { - assert.ifError(err); + const override = { + aggregation: 'day', + offset: '-3600' + }; - var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 - var initialTimestamp = '1969-12-31T00:00:00-01:00'; - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(OFFSET_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); + this.testClient.getDataview(test.dataviewId, override, function(err, dataview) { + assert.ifError(err); - dataview.bins.forEach(function (bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(OFFSET_IN_MINUTES) - .add(index, override.aggregation) - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(OFFSET_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); }); }); From 2024e89c6ab227091e680d5cf74c480ccaf1795d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 18:30:17 +0200 Subject: [PATCH 329/402] Update news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 3d5d5722..ced38066 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,7 @@ Announcements: Bug fixes: - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + - Support timestamp with timezones to calculate the number of bins in time-series. ## 3.10.1 From b2bbc329ea9a542925644f5e3a6b540ffb6e2fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:03:49 +0200 Subject: [PATCH 330/402] Apply prefix for intermediate query variables to avoid name colision --- lib/cartodb/models/dataview/histogram.js | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index a044bfc7..6d399d39 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -176,14 +176,14 @@ var histogramQueryTpl = dot.template([ var dateBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', - ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', - ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(date_trunc(', ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )) AS start_date,', - ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS end_date,', - ' count(1) AS total_rows', + ' )) AS __cdb_start_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' ].join(' \n')); @@ -191,19 +191,19 @@ var dateBasicsQueryTpl = dot.template([ var dateOverrideBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val,', - ' min({{=it._start}}) AS min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' max({{=it._end}}) AS __cdb_max_val,', + ' min({{=it._start}}) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(', ' date_trunc(', ' \'{{=it._aggregation}}\',', ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )', - ' ) AS start_date,', + ' ) AS __cdb_start_date,', ' max(', ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AS end_date,', - ' count(1) AS total_rows', + ' ) AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' ].join(' \n')); @@ -217,8 +217,8 @@ var dateBinsQueryTpl = dot.template([ ' SELECT', ' ARRAY(', ' SELECT GENERATE_SERIES(', - ' start_date::timestamptz,', - ' end_date::timestamptz,', + ' __cdb_start_date::timestamptz,', + ' __cdb_end_date::timestamptz,', ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', ' ) AS bins_array', @@ -229,10 +229,10 @@ var dateBinsQueryTpl = dot.template([ var dateHistogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' (__cdb_max_val - __cdb_min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', - ' CASE WHEN min_val = max_val', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', @@ -250,14 +250,14 @@ var dateHistogramQueryTpl = dot.template([ ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', - ' date_part(\'epoch\', start_date)::numeric AS timestamp_start,', + ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, __cdb_start_date', 'ORDER BY bin' ].join('\n')); From 0f45675652a87488ba261e0c1035fed650d6ed78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:06:31 +0200 Subject: [PATCH 331/402] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index ced38066..bd89d748 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ Announcements: Bug fixes: - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. - Support timestamp with timezones to calculate the number of bins in time-series. + - Fixed issue related to name collision while building time-series query. ## 3.10.1 From 5c80ff81917f4093a1ac9c3e3dc3964b4b17337a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:24:15 +0200 Subject: [PATCH 332/402] Extract query: move condition to a method --- lib/cartodb/models/dataview/histogram.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6d399d39..4af16d65 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -328,12 +328,16 @@ Histogram.prototype.sql = function(psql, override, callback) { this._buildQuery(psql, override, callback); }; +Histogram.prototype.isDateHistogram = function (override) { + return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); +}; + Histogram.prototype._buildQuery = function (psql, override, callback) { var filteredQuery, basicsQuery, binsQuery; var _column = this.column; var _query = this.query; - if (this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined)) { + if (this.isDateHistogram(override)) { return this._buildDateHistogramQuery(psql, override, callback); } From 994e58bef7e2f9f63a66b6c04e4bf9d38428915f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:33:26 +0200 Subject: [PATCH 333/402] Add prefix to bins_number and nulls_cout to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4af16d65..dce22bb0 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -109,7 +109,7 @@ var binsQueryTpl = dot.template([ ' {{=it._maxBins}}', ' )', ' )', - ' END AS bins_number', + ' END AS __cdb_bins_number', ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' @@ -117,14 +117,14 @@ var binsQueryTpl = dot.template([ var overrideBinsQueryTpl = dot.template([ 'bins AS (', - ' SELECT {{=it._bins}} AS bins_number', + ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); var nullsQueryTpl = dot.template([ 'nulls AS (', ' SELECT', - ' count(*) AS nulls_count', + ' count(*) AS __cdb_nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' @@ -153,22 +153,22 @@ var nansQueryTpl = dot.template([ var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', + ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', ' {{?it._isFloatColumn}}infinities_count,', ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number), __cdb_bins_number)) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', - 'GROUP BY bin, bins_number, bin_width, nulls_count,', + 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -212,7 +212,7 @@ var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', ' bins_array,', - ' ARRAY_LENGTH(bins_array, 1) AS bins_number', + ' ARRAY_LENGTH(bins_array, 1) AS __cdb_bins_number', ' FROM (', ' SELECT', ' ARRAY(', @@ -229,9 +229,9 @@ var dateBinsQueryTpl = dot.template([ var dateHistogramQueryTpl = dot.template([ 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', @@ -239,7 +239,7 @@ var dateHistogramQueryTpl = dot.template([ ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', ' bins_array', ' ),', - ' bins_number', + ' __cdb_bins_number', ' )) - 1', ' END AS bin,', ' min(', @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, __cdb_start_date', + 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count, __cdb_avg_val, __cdb_start_date', 'ORDER BY bin' ].join('\n')); From 823925d09143b75e96c1f9d308bba741ce7ecf4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:41:22 +0200 Subject: [PATCH 334/402] Add prefix to bins_array to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dce22bb0..c6f4ffbb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -211,8 +211,8 @@ var dateOverrideBasicsQueryTpl = dot.template([ var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', - ' bins_array,', - ' ARRAY_LENGTH(bins_array, 1) AS __cdb_bins_number', + ' __cdb_bins_array,', + ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', ' FROM (', ' SELECT', ' ARRAY(', @@ -221,9 +221,9 @@ var dateBinsQueryTpl = dot.template([ ' __cdb_end_date::timestamptz,', ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', - ' ) AS bins_array', + ' ) AS __cdb_bins_array', ' FROM basics', - ' ) _cdb_bins_array', + ' ) _cdb_bins_array_query', ')' ].join('\n')); @@ -237,7 +237,7 @@ var dateHistogramQueryTpl = dot.template([ ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', - ' bins_array', + ' __cdb_bins_array', ' ),', ' __cdb_bins_number', ' )) - 1', From e4461601512b40f599e62e97221d6c8690d334ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:45:05 +0200 Subject: [PATCH 335/402] Use final columns to group by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c6f4ffbb..d868513a 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count, __cdb_avg_val, __cdb_start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, timestamp_start', 'ORDER BY bin' ].join('\n')); From 91d5a0e4e4c38540606ac5bf322c02a2fe360c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:46:26 +0200 Subject: [PATCH 336/402] Remove column avg_val in group_by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index d868513a..46f86a5d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, timestamp_start', + 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' ].join('\n')); From 0f8de9e74b920d0d8ab20ce379bf930c44011756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:06:36 +0200 Subject: [PATCH 337/402] Add prefix to basics query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 28 ++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 46f86a5d..c9204960 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -67,7 +67,7 @@ var filteredQueryTpl = dot.template([ ].join(' \n')); var basicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', @@ -76,7 +76,7 @@ var basicsQueryTpl = dot.template([ ].join(' \n')); var overrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', @@ -110,7 +110,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS __cdb_bins_number', - ' FROM basics, iqrange, filtered_source', + ' FROM __cdb_basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -161,20 +161,26 @@ var histogramQueryTpl = dot.template([ ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number), __cdb_bins_number)) - 1', + ' ELSE GREATEST(', + ' 1,', + ' LEAST(', + ' WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number),', + ' __cdb_bins_number', + ' )', + ' ) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); var dateBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', @@ -184,12 +190,12 @@ var dateBasicsQueryTpl = dot.template([ ' )) AS __cdb_start_date,', ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM ({{=it._query}}) __cdb_basics_query', ')' ].join(' \n')); var dateOverrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._end}}) AS __cdb_max_val,', ' min({{=it._start}}) AS __cdb_min_val,', @@ -204,7 +210,7 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AS __cdb_end_date,', ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM ({{=it._query}}) __cdb_basics_query', ')' ].join(' \n')); @@ -222,7 +228,7 @@ var dateBinsQueryTpl = dot.template([ ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', ' ) AS __cdb_bins_array', - ' FROM basics', + ' FROM __cdb_basics', ' ) _cdb_bins_array_query', ')' ].join('\n')); @@ -255,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From f3ababffc1f29e8f2fb8e5fed34b158888e56429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:12:52 +0200 Subject: [PATCH 338/402] Prefix bins query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c9204960..2af66449 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -99,7 +99,7 @@ var iqrQueryTpl = dot.template([ ].join('\n')); var binsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', ' THEN 1', ' ELSE GREATEST(', @@ -116,7 +116,7 @@ var binsQueryTpl = dot.template([ ].join('\n')); var overrideBinsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' @@ -215,7 +215,7 @@ var dateOverrideBasicsQueryTpl = dot.template([ ].join(' \n')); var dateBinsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT', ' __cdb_bins_array,', ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', @@ -261,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From 3ee064a59fa87cb2d0802568af1fd8a9d190d465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:32:13 +0200 Subject: [PATCH 339/402] Prefix nulls query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 2af66449..54a9fcf6 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -122,7 +122,7 @@ var overrideBinsQueryTpl = dot.template([ ].join('\n')); var nullsQueryTpl = dot.template([ - 'nulls AS (', + '__cdb_nulls AS (', ' SELECT', ' count(*) AS __cdb_nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' @@ -261,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From a9e3bc3cdadfbfd1d57cf609442096499f1100c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:36:23 +0200 Subject: [PATCH 340/402] Prefix filtered source to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 54a9fcf6..629a69f7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -52,9 +52,9 @@ var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; var filteredQueryTpl = dot.template([ - 'filtered_source AS (', + '__cdb_filtered_source AS (', ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source', + ' FROM ({{=it._query}}) _cdb_filtered_source_query', ' WHERE', ' {{=it._column}} IS NOT NULL', ' {{?it._isFloatColumn}}AND', @@ -71,7 +71,7 @@ var basicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' FROM __cdb_filtered_source', ')' ].join(' \n')); @@ -80,7 +80,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' FROM __cdb_filtered_source', ')' ].join('\n')); @@ -91,7 +91,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM filtered_source) _cdb_quartiles', + ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -110,7 +110,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS __cdb_bins_number', - ' FROM __cdb_basics, iqrange, filtered_source', + ' FROM __cdb_basics, iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From 7e058955ea71ac88010855c3d3d03fc56226e22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:39:51 +0200 Subject: [PATCH 341/402] Use final naming to group by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 629a69f7..7099340d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -174,7 +174,7 @@ var histogramQueryTpl = dot.template([ ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', - 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', + 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); From 34cf45bc9dedbfbc4595ac1e7a8151b606e90716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:43:31 +0200 Subject: [PATCH 342/402] Prefix infinities query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7099340d..0edfaae9 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -131,7 +131,7 @@ var nullsQueryTpl = dot.template([ ].join('\n')); var infinitiesQueryTpl = dot.template([ - 'infinities AS (', + '__cdb_infinities AS (', ' SELECT', ' count(*) AS infinities_count', ' FROM ({{=it._query}}) _cdb_histogram_infinities', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From c7e5dbf1585a541fe2b5328f576896770ecc691d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:46:34 +0200 Subject: [PATCH 343/402] Fix query aliases --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 0edfaae9..01335421 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -134,7 +134,7 @@ var infinitiesQueryTpl = dot.template([ '__cdb_infinities AS (', ' SELECT', ' count(*) AS infinities_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' FROM ({{=it._query}}) __cdb_infinities_query', ' WHERE', ' {{=it._column}} = \'infinity\'::float', ' OR', @@ -146,7 +146,7 @@ var nansQueryTpl = dot.template([ 'nans AS (', ' SELECT', ' count(*) AS nans_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' FROM ({{=it._query}}) __cdb_nans_query', ' WHERE {{=it._column}} = \'NaN\'::float', ')' ].join('\n')); From cf71489a7f3c18718e0aefdd5c5b84997bc26960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:54:49 +0200 Subject: [PATCH 344/402] Prefix nans and infinities counters to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 01335421..26ee452c 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -133,7 +133,7 @@ var nullsQueryTpl = dot.template([ var infinitiesQueryTpl = dot.template([ '__cdb_infinities AS (', ' SELECT', - ' count(*) AS infinities_count', + ' count(*) AS __cdb_infinities_count', ' FROM ({{=it._query}}) __cdb_infinities_query', ' WHERE', ' {{=it._column}} = \'infinity\'::float', @@ -143,9 +143,9 @@ var infinitiesQueryTpl = dot.template([ ].join('\n')); var nansQueryTpl = dot.template([ - 'nans AS (', + '__cdb_nans AS (', ' SELECT', - ' count(*) AS nans_count', + ' count(*) AS __cdb_nans_count', ' FROM ({{=it._query}}) __cdb_nans_query', ' WHERE {{=it._column}} = \'NaN\'::float', ')' @@ -156,8 +156,8 @@ var histogramQueryTpl = dot.template([ ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', ' __cdb_bins_number AS bins_number,', ' __cdb_nulls_count AS nulls_count,', - ' {{?it._isFloatColumn}}infinities_count,', - ' nans_count,{{?}}', + ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', + ' __cdb_nans_count AS nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From ff3d7ed7b21829194518a2d75a84c80d3d4bc126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:57:12 +0200 Subject: [PATCH 345/402] Fix jshint typo --- lib/cartodb/models/dataview/histogram.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 26ee452c..ec3e616e 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -173,7 +173,8 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', + ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From 7e159c565b61ffdb2bfdbe3e385b5d6daae8541d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:03:40 +0200 Subject: [PATCH 346/402] Prefix iqr query calculation to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index ec3e616e..9c63b836 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -85,8 +85,8 @@ var overrideBasicsQueryTpl = dot.template([ ].join('\n')); var iqrQueryTpl = dot.template([ - 'iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + '__cdb_iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', ' FROM (', ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', @@ -100,17 +100,17 @@ var iqrQueryTpl = dot.template([ var binsQueryTpl = dot.template([ '__cdb_bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + ' SELECT CASE WHEN total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((max_val - min_val) / (2 * __cdb_iqr * power(total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', ' END AS __cdb_bins_number', - ' FROM __cdb_basics, iqrange, __cdb_filtered_source', + ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); From 3c061769c68582eb645a8e6184bbd3cedd13046e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:11:35 +0200 Subject: [PATCH 347/402] Prefix basics columns to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 9c63b836..7d29d01d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -69,8 +69,8 @@ var filteredQueryTpl = dot.template([ var basicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', ' FROM __cdb_filtered_source', ')' ].join(' \n')); @@ -78,8 +78,8 @@ var basicsQueryTpl = dot.template([ var overrideBasicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', ' FROM __cdb_filtered_source', ')' ].join('\n')); @@ -100,12 +100,12 @@ var iqrQueryTpl = dot.template([ var binsQueryTpl = dot.template([ '__cdb_bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR __cdb_iqr = 0', + ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * __cdb_iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', @@ -153,18 +153,18 @@ var nansQueryTpl = dot.template([ var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', ' __cdb_bins_number AS bins_number,', ' __cdb_nulls_count AS nulls_count,', ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', ' __cdb_nans_count AS nans_count,{{?}}', - ' avg_val,', - ' CASE WHEN min_val = max_val', + ' __cdb_avg_val AS avg_val,', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(', ' 1,', ' LEAST(', - ' WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number),', + ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', ' __cdb_bins_number', ' )', ' ) - 1', From e7a0b246a39cd722138196c68b657b94bd1154e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:20:36 +0200 Subject: [PATCH 348/402] Prefix with double underscore --- lib/cartodb/models/dataview/histogram.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7d29d01d..b07aa474 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -54,7 +54,7 @@ var BIN_MAX_NUMBER = 48; var filteredQueryTpl = dot.template([ '__cdb_filtered_source AS (', ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source_query', + ' FROM ({{=it._query}}) __cdb_filtered_source_query', ' WHERE', ' {{=it._column}} IS NOT NULL', ' {{?it._isFloatColumn}}AND', @@ -94,7 +94,7 @@ var iqrQueryTpl = dot.template([ ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', - ' ) _cdb_iqr', + ' ) __cdb_iqr', ')' ].join('\n')); @@ -125,7 +125,7 @@ var nullsQueryTpl = dot.template([ '__cdb_nulls AS (', ' SELECT', ' count(*) AS __cdb_nulls_count', - ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' FROM ({{=it._query}}) __cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' ].join('\n')); @@ -230,7 +230,7 @@ var dateBinsQueryTpl = dot.template([ ' )', ' ) AS __cdb_bins_array', ' FROM __cdb_basics', - ' ) _cdb_bins_array_query', + ' ) __cdb_bins_array_query', ')' ].join('\n')); From c643160671b5a40dec233192d33a6aaafc0667e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:32:53 +0200 Subject: [PATCH 349/402] Prefix date interval query (to calculate automatic aggregation) to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 50 ++++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b07aa474..c27c696f 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -9,42 +9,42 @@ var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var dateIntervalQueryTpl = dot.template([ 'WITH', - 'dates AS (', + '__cdb_dates AS (', ' SELECT', - ' MAX({{=it.column}}::timestamp) AS _end,', - ' MIN({{=it.column}}::timestamp) AS _start', - ' FROM ({{=it.query}}) _cdb_source', + ' MAX({{=it.column}}::timestamp) AS __cdb_end,', + ' MIN({{=it.column}}::timestamp) AS __cdb_start', + ' FROM ({{=it.query}}) __cdb_source', '),', - 'interval_in_days AS (', + '__cdb_interval_in_days AS (', ' SELECT' , - ' DATE_PART(\'day\', _end - _start) AS days', - ' FROM dates', + ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', + ' FROM __cdb_dates', '),', - 'interval_in_hours AS (', + '__cdb_interval_in_hours AS (', ' SELECT', - ' days * 24 + DATE_PART(\'hour\', _end - _start) AS hours', - ' FROM interval_in_days, dates', + ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', + ' FROM __cdb_interval_in_days, __cdb_dates', '),', - 'interval_in_minutes AS (', + '__cdb_interval_in_minutes AS (', ' SELECT', - ' hours * 60 + DATE_PART(\'minute\', _end - _start) AS minutes', - ' FROM interval_in_hours, dates', + ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', + ' FROM __cdb_interval_in_hours, __cdb_dates', '),', - 'interval_in_seconds AS (', + '__cdb_interval_in_seconds AS (', ' SELECT', - ' minutes * 60 + DATE_PART(\'second\', _end - _start) AS seconds', - ' FROM interval_in_minutes, dates', + ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', + ' FROM __cdb_interval_in_minutes, __cdb_dates', ')', 'SELECT', - ' ROUND(days / 365) AS year,', - ' ROUND(days / 90) AS quarter,', - ' ROUND(days / 30) AS month,', - ' ROUND(days / 7) AS week,', - ' days AS day,', - ' hours AS hour,', - ' minutes AS minute,', - ' seconds AS second', - 'FROM interval_in_days, interval_in_hours, interval_in_minutes, interval_in_seconds' + ' ROUND(__cdb_days / 365) AS year,', + ' ROUND(__cdb_days / 90) AS quarter,', + ' ROUND(__cdb_days / 30) AS month,', + ' ROUND(__cdb_days / 7) AS week,', + ' __cdb_days AS day,', + ' __cdb_hours AS hour,', + ' __cdb_minutes AS minute,', + ' __cdb_seconds AS second', + 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' ].join('\n')); var MAX_INTERVAL_VALUE = 366; From b24858a17cb4d89c672573779ef7096131a38db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 11:21:44 +0200 Subject: [PATCH 350/402] Release 3.11.0 --- NEWS.md | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index bd89d748..1f5f64c7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog -## 3.10.2 -Released 2017-mm-dd +## 3.11.0 +Released 2017-08-08 Announcements: - Allow to override with any aggregation for histograms instantiated w/o aggregation. diff --git a/package.json b/package.json index df934d7a..898707a7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.10.2", + "version": "3.11.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 3611752677e8bdbef7a94f338c632d0a04ec67b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 11:25:18 +0200 Subject: [PATCH 351/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 1f5f64c7..bf1dd7e4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.11.1 +Released 2017-mm-dd + + ## 3.11.0 Released 2017-08-08 diff --git a/package.json b/package.json index 898707a7..a3e4416c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.11.0", + "version": "3.11.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From d27a2810671f45ab0ac1b75ab9f3a17535ad8c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 Aug 2017 17:55:39 +0200 Subject: [PATCH 352/402] Upgrade cartodb-redis to 0.14.0 --- package.json | 2 +- yarn.lock | 74 ++++++++++++++++++++++++++-------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 093a6948..212689cb 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "camshaft": "0.55.6", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", - "cartodb-redis": "cartodb/node-cartodb-redis#timeout-limits", + "cartodb-redis": "0.14.0", "debug": "~2.2.0", "dot": "~1.0.2", "express": "~4.13.3", diff --git a/yarn.lock b/yarn.lock index ebc2c0d0..74f6d935 100644 --- a/yarn.lock +++ b/yarn.lock @@ -223,6 +223,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -231,14 +239,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#0.15.1-cdb1": - version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -258,9 +258,9 @@ cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" -cartodb-redis@cartodb/node-cartodb-redis#timeout-limits: - version "0.13.3" - resolved "https://codeload.github.com/cartodb/node-cartodb-redis/tar.gz/4c0586009dbfd464eb9e742bed39170334f40453" +cartodb-redis@0.14.0: + version "0.14.0" + resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.14.0.tgz#6f82fdb3e5b7c8005dbaccd6172c1706c4378df2" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" @@ -1741,7 +1741,32 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.81.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1768,31 +1793,6 @@ request@2.x, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" From ce97844f37e29816255e991548b65610946f3ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 Aug 2017 18:42:44 +0200 Subject: [PATCH 353/402] Upgrade windshaft version to 3.3.0 --- package.json | 2 +- yarn.lock | 108 +++++++++++++++++++++++++++------------------------ 2 files changed, 59 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index 212689cb..fe80d464 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#response-timeout-limit", + "windshaft": "3.3.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 74f6d935..e6aa6bce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -254,6 +254,14 @@ cartodb-psql@0.8.0: step "~0.0.6" underscore "~1.6.0" +cartodb-psql@0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.9.0.tgz#887ba006fb499ab789c15d10f34f6afbe1f2f68d" + dependencies: + debug "~2.2.0" + pg cartodb/node-postgres#6.1.2-cdb1 + underscore "~1.6.0" + cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" @@ -1312,7 +1320,7 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0: +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" @@ -1343,7 +1351,7 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: +node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36: version "0.6.36" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" dependencies: @@ -1520,7 +1528,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1741,32 +1749,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.81.0: +request@2.x, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1793,6 +1776,31 @@ request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@^2.55.0, request@^2.69.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1964,11 +1972,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.8" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.8.tgz#4cbcf965d8b901d1b1015cbc7fc415aae157dfaa" + version "3.1.9" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.9.tgz#b2e7fbaa348380318d3834323918c3c351b8bf18" dependencies: - nan "~2.4.0" - node-pre-gyp "~0.6.31" + nan "~2.6.2" + node-pre-gyp "~0.6.36" srs@1.x: version "1.2.0" @@ -2103,17 +2111,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#query-timeout: - version "2.3.1-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/38d5d82355aa9eef68c29b7d7a33b60a3ae394f7" +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": + version "2.3.1-cdb4" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#custom-timeout: - version "0.6.18-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/f9042e5eab92e26fb9e2a6ace65af92cc176a218" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": + version "0.6.18-cdb3" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2274,14 +2282,14 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#response-timeout-limit: - version "3.2.2" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/21ebe5f69b5b6215ec2f93088d8eee436f6b81d5" +windshaft@3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.0.tgz#bf8ef35f9a2bb126934531b0020f5e8f96b37dfa" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.8.0" + cartodb-psql "0.9.0" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" @@ -2293,8 +2301,8 @@ windshaft@cartodb/windshaft#response-timeout-limit: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#query-timeout - tilelive-mapnik cartodb/tilelive-mapnik#custom-timeout + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb4 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb3 torque.js "~2.11.0" underscore "~1.6.0" From ff560ffde7079d4e39c7b96cb95a33ae623e391d Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 9 Aug 2017 18:49:59 +0200 Subject: [PATCH 354/402] add test boundingBox-polygon-counter --- test/acceptance/dataviews/polygonCount.js | 79 +++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 test/acceptance/dataviews/polygonCount.js diff --git a/test/acceptance/dataviews/polygonCount.js b/test/acceptance/dataviews/polygonCount.js new file mode 100644 index 00000000..02ff075e --- /dev/null +++ b/test/acceptance/dataviews/polygonCount.js @@ -0,0 +1,79 @@ +require('../../support/test_helper'); +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + +describe('boundingBox-polygon-counter', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_formula: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: "cartodb_id", + operation: "count", + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": ` + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-161.015625,69.28725695167886],[-162.7734375,-7.710991655433217],[-40.78125,-8.059229627200192],[-161.015625,69.28725695167886]]]}'), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id + UNION ALL + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-29.179687499999996,-7.01366792756663],[103.71093749999999,-6.664607562172573],[105.46875,69.16255790810501],[-29.179687499999996,-7.01366792756663]]]}'), 4326), 3857), 2 + UNION ALL + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-117.42187500000001,68.13885164925573],[-35.859375,20.96143961409684],[59.4140625,68.52823492039876],[-117.42187500000001,68.13885164925573]]]}'), 4326), 3857), 3 + ` + } + } + ] + ); + + it('should not count the polygons outside the bounding box', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + params = { + bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' + } + this.testClient.getDataview('val_formula', params, function(err, dataview) { + assert.equal(dataview.result, 1); + done(); + }); + }); +}); \ No newline at end of file From 814b123b2bc594dd1aeddcaad1002af5aeeb1e8f Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 9 Aug 2017 18:55:14 +0200 Subject: [PATCH 355/402] fix 725 using the ST_Intersects function instead of && --- lib/cartodb/models/filter/bbox.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js index dcf19e55..0f7072ae 100644 --- a/lib/cartodb/models/filter/bbox.js +++ b/lib/cartodb/models/filter/bbox.js @@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([ ].join('\n')); var bboxFilterTpl = dot.template( - '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})' + 'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))' ); var LATITUDE_MAX_VALUE = 85.0511287798066; From 44c5eb051d692cd3c051d88812ccefa166a057a7 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 11:05:36 +0200 Subject: [PATCH 356/402] formatting the query of polygon count test --- test/acceptance/dataviews/polygonCount.js | 46 ++++++++++++++++++++--- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/test/acceptance/dataviews/polygonCount.js b/test/acceptance/dataviews/polygonCount.js index 02ff075e..02ceef28 100644 --- a/test/acceptance/dataviews/polygonCount.js +++ b/test/acceptance/dataviews/polygonCount.js @@ -21,6 +21,36 @@ describe('boundingBox-polygon-counter', function() { } }); + var polygon1 = { + type: "Polygon", + coordinates:[[ + [-161.015625,69.28725695167886], + [-162.7734375,-7.710991655433217], + [-40.78125,-8.059229627200192], + [-161.015625,69.28725695167886] + ]] + }; + + var polygon2 = { + type: "Polygon", + coordinates: [[ + [-29.179687499999996,-7.01366792756663], + [103.71093749999999,-6.664607562172573], + [105.46875,69.16255790810501], + [-29.179687499999996,-7.01366792756663] + ]] + }; + + var polygon3 = { + type: "Polygon", + coordinates:[[ + [-117.42187500000001,68.13885164925573], + [-35.859375,20.96143961409684], + [59.4140625,68.52823492039876], + [-117.42187500000001,68.13885164925573] + ]] + }; + var mapConfig = createMapConfig( [ { @@ -53,13 +83,19 @@ describe('boundingBox-polygon-counter', function() { "params": { "query": ` SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-161.015625,69.28725695167886],[-162.7734375,-7.710991655433217],[-40.78125,-8.059229627200192],[-161.015625,69.28725695167886]]]}'), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(polygon1)}' + ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id UNION ALL SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-29.179687499999996,-7.01366792756663],[103.71093749999999,-6.664607562172573],[105.46875,69.16255790810501],[-29.179687499999996,-7.01366792756663]]]}'), 4326), 3857), 2 + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(polygon2)}' + ), 4326), 3857), 2 UNION ALL SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-117.42187500000001,68.13885164925573],[-35.859375,20.96143961409684],[59.4140625,68.52823492039876],[-117.42187500000001,68.13885164925573]]]}'), 4326), 3857), 3 + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(polygon3)}' + ), 4326), 3857), 3 ` } } @@ -68,9 +104,9 @@ describe('boundingBox-polygon-counter', function() { it('should not count the polygons outside the bounding box', function(done) { this.testClient = new TestClient(mapConfig, 1234); - params = { + var params = { bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' - } + }; this.testClient.getDataview('val_formula', params, function(err, dataview) { assert.equal(dataview.result, 1); done(); From 23edf78a67e5c87753059bd3eeb989c5b063dbfd Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 15:58:25 +0200 Subject: [PATCH 357/402] Remove unnecessary step --- lib/cartodb/api/user_limits_api.js | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 74d0cf21..0f4f0f4c 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -23,23 +23,19 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) render: self.options.limits.render || 0 }; - step( - function getTimeoutLimit() { - self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { - if (err) { - return callback(err); - } - - if (timeoutRenderLimit && timeoutRenderLimit.render) { - if (Number.isFinite(timeoutRenderLimit.render)) { - limits.render = timeoutRenderLimit.render; - } - } - - return callback(null, limits); - }); + self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { + if (err) { + return callback(err); } - ); + + if (timeoutRenderLimit && timeoutRenderLimit.render) { + if (Number.isFinite(timeoutRenderLimit.render)) { + limits.render = timeoutRenderLimit.render; + } + } + + return callback(null, limits); + }); }; UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { From 69eaa72819a0efd0d9c0a8a1e9c0115cb6aca2fc Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:06:10 +0200 Subject: [PATCH 358/402] String comparison and regex to match errors instead of indexOf --- lib/cartodb/controllers/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 98c5ee2f..f74702e9 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -301,11 +301,11 @@ function statusFromErrorMessage(errMsg) { } function isRenderTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('Render timed out')); + return err.message === 'Render timed out'; } function isDatasourceTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); + return err.message && err.message.match(/canceling statement due to statement timeout/i); } function isTimeoutError (err) { From e7c206762dda88ea4e3b7322cad70ccca31da6e2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:09:26 +0200 Subject: [PATCH 359/402] String comparison and regex to match errors instead of indexOf --- lib/cartodb/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7aea23cd..d9302276 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -120,11 +120,11 @@ module.exports = function(serverOptions) { onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { function isRenderTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('Render timed out')); + return err.message === 'Render timed out'; } function isDatasourceTimeoutError (err) { - return err.message && (-1 !== err.message.indexOf('canceling statement due to statement timeout')); + return err.message && err.message.match(/canceling statement due to statement timeout/i); } function isTimeoutError (err) { From 14d5ee417881206bb7b73e60cfcdf8aab040e67a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:22:31 +0200 Subject: [PATCH 360/402] Remove user param --- .../acceptance/user-database-timeout-limit.js | 44 +++++++++---------- test/support/test-client.js | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index e1ff7bda..03a5e0c5 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -81,11 +81,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -118,11 +118,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -157,11 +157,11 @@ describe('user database timeout limit', function () { describe('fetching raster tiles', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); describe('with onTileErrorStrategy ENABLED', function () { @@ -326,11 +326,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -390,11 +390,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('"mvt" fails due to statement timeout', function (done) { @@ -428,11 +428,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -492,11 +492,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('"grid.json" fails due to statement timeout', function (done) { @@ -532,11 +532,11 @@ describe('user database timeout limit', function () { cartocss: TestClient.CARTOCSS.TORQUE }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -599,11 +599,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('"torque.json" fails due to statement timeout', function (done) { @@ -663,11 +663,11 @@ describe('user database timeout limit', function () { } }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, (err) => { + TestClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -738,11 +738,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 200, done); + TestClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit('localhost', 0, done); + TestClient.setUserDatabaseTimeoutLimit(0, done); }); it('fails due to statement timeout', function (done) { diff --git a/test/support/test-client.js b/test/support/test-client.js index 196e1a8e..3be8bf71 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1164,7 +1164,7 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) { +TestClient.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) From 95244334374d0fe31b7b2927d740294988926d1d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Aug 2017 16:24:40 +0200 Subject: [PATCH 361/402] Use instance server --- test/support/test-client.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 3be8bf71..1f8344df 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -487,7 +487,7 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -526,7 +526,7 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId; - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', From 01a22a45bb3973e86dba433773d2507286a958d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 10 Aug 2017 17:09:05 +0200 Subject: [PATCH 362/402] Move `setUserDatabaseTimeoutLimit` from class method to a instance method --- .../acceptance/user-database-timeout-limit.js | 88 +++++++++++-------- test/support/test-client.js | 2 +- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 03a5e0c5..42c468fd 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -81,11 +81,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -118,11 +118,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -156,14 +156,6 @@ describe('user database timeout limit', function () { describe('fetching raster tiles', function () { describe('with user\'s timeout of 200 ms', function () { - beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); - }); - - afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); - }); - describe('with onTileErrorStrategy ENABLED', function () { let onTileErrorStrategy; @@ -180,21 +172,31 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.setUserDatabaseTimeoutLimit(200, (err) => { if (err) { return done(err); } - this.layergroupid = res.layergroupid; + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - done(); + this.layergroupid = res.layergroupid; + + done(); + }); }); }); afterEach(function (done) { global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - - this.testClient.drain(done); + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); }); it('"png" fails due to statement timeout', function (done) { @@ -252,21 +254,33 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.setUserDatabaseTimeoutLimit(200, (err) => { if (err) { return done(err); } - this.layergroupid = res.layergroupid; + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } - done(); + this.layergroupid = res.layergroupid; + + done(); + }); }); }); afterEach(function (done) { global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; - this.testClient.drain(done); + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + + this.testClient.drain(done); + }); }); it('"png" fails due to statement timeout', function (done) { @@ -326,11 +340,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -390,11 +404,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('"mvt" fails due to statement timeout', function (done) { @@ -428,11 +442,11 @@ describe('user database timeout limit', function () { beforeEach(function (done) { const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -492,11 +506,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('"grid.json" fails due to statement timeout', function (done) { @@ -532,11 +546,11 @@ describe('user database timeout limit', function () { cartocss: TestClient.CARTOCSS.TORQUE }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -599,11 +613,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('"torque.json" fails due to statement timeout', function (done) { @@ -663,11 +677,11 @@ describe('user database timeout limit', function () { } }); this.testClient = new TestClient(mapconfig, 1234); - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { if (err) { return done(err); } @@ -738,11 +752,11 @@ describe('user database timeout limit', function () { describe('with user\'s timeout of 200 ms', function () { beforeEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(200, done); + this.testClient.setUserDatabaseTimeoutLimit(200, done); }); afterEach(function (done) { - TestClient.setUserDatabaseTimeoutLimit(0, done); + this.testClient.setUserDatabaseTimeoutLimit(0, done); }); it('fails due to statement timeout', function (done) { diff --git a/test/support/test-client.js b/test/support/test-client.js index 1f8344df..21918665 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1164,7 +1164,7 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; -TestClient.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { +TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) From e1e22de65fe39da07924bbc441a74c54272c05fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 10 Aug 2017 17:52:46 +0200 Subject: [PATCH 363/402] Release 3.12.0 --- NEWS.md | 9 +++++++-- package.json | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index bf1dd7e4..2a7fdc46 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,12 @@ # Changelog -## 3.11.1 -Released 2017-mm-dd +## 3.12.0 +Released 2017-08-10 + +Announcements: + - Apply max tile response time for requests to layergoup, tiles, static maps, attributes and dataviews services #717. + - Upgrades windshaft to [3.3.0](https://github.com/CartoDB/windshaft/releases/tag/3.3.0). + - Upgrades cartodb-redis to [0.14.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/0.14.0). ## 3.11.0 diff --git a/package.json b/package.json index d0e0be6b..f482fffb 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.11.1", + "version": "3.12.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From e678957a8f0536abb744a99c767182eb2510c152 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 18:09:18 +0200 Subject: [PATCH 364/402] move polygon count test to widgets regression testfile, and check the only returned polygon is the expected one --- test/acceptance/dataviews/polygonCount.js | 115 ---------------------- test/acceptance/widgets/regressions.js | 97 ++++++++++++++++++ 2 files changed, 97 insertions(+), 115 deletions(-) delete mode 100644 test/acceptance/dataviews/polygonCount.js diff --git a/test/acceptance/dataviews/polygonCount.js b/test/acceptance/dataviews/polygonCount.js deleted file mode 100644 index 02ceef28..00000000 --- a/test/acceptance/dataviews/polygonCount.js +++ /dev/null @@ -1,115 +0,0 @@ -require('../../support/test_helper'); -var assert = require('../../support/assert'); -var TestClient = require('../../support/test-client'); - -function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; -} - -describe('boundingBox-polygon-counter', function() { - - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - - var polygon1 = { - type: "Polygon", - coordinates:[[ - [-161.015625,69.28725695167886], - [-162.7734375,-7.710991655433217], - [-40.78125,-8.059229627200192], - [-161.015625,69.28725695167886] - ]] - }; - - var polygon2 = { - type: "Polygon", - coordinates: [[ - [-29.179687499999996,-7.01366792756663], - [103.71093749999999,-6.664607562172573], - [105.46875,69.16255790810501], - [-29.179687499999996,-7.01366792756663] - ]] - }; - - var polygon3 = { - type: "Polygon", - coordinates:[[ - [-117.42187500000001,68.13885164925573], - [-35.859375,20.96143961409684], - [59.4140625,68.52823492039876], - [-117.42187500000001,68.13885164925573] - ]] - }; - - var mapConfig = createMapConfig( - [ - { - "type": "cartodb", - "options": { - "source": { - "id": "a0" - }, - "cartocss": "#points { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0" - } - } - ], - { - val_formula: { - source: { - id: 'a0' - }, - type: 'formula', - options: { - column: "cartodb_id", - operation: "count", - } - } - }, - [ - { - "id": "a0", - "type": "source", - "params": { - "query": ` - SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( - '${JSON.stringify(polygon1)}' - ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id - UNION ALL - SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( - '${JSON.stringify(polygon2)}' - ), 4326), 3857), 2 - UNION ALL - SELECT - ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( - '${JSON.stringify(polygon3)}' - ), 4326), 3857), 3 - ` - } - } - ] - ); - - it('should not count the polygons outside the bounding box', function(done) { - this.testClient = new TestClient(mapConfig, 1234); - var params = { - bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' - }; - this.testClient.getDataview('val_formula', params, function(err, dataview) { - assert.equal(dataview.result, 1); - done(); - }); - }); -}); \ No newline at end of file diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 1d006ac4..f3ab3aa8 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -218,6 +218,103 @@ describe('widgets-regressions', function() { }); }); + + it('should not count the polygons outside the bounding box', function(done) { + var notIntersectingLeftTriangle = { + type: "Polygon", + coordinates:[[ + [-161.015625,69.28725695167886], + [-162.7734375,-7.710991655433217], + [-40.78125,-8.059229627200192], + [-161.015625,69.28725695167886] + ]] + }; + + var notIntersectingRightTriangle = { + type: "Polygon", + coordinates: [[ + [-29.179687499999996,-7.01366792756663], + [103.71093749999999,-6.664607562172573], + [105.46875,69.16255790810501], + [-29.179687499999996,-7.01366792756663] + ]] + }; + + var intersectingTriangle = { + type: "Polygon", + coordinates:[[ + [-117.42187500000001,68.13885164925573], + [-35.859375,20.96143961409684], + [59.4140625,68.52823492039876], + [-117.42187500000001,68.13885164925573] + ]] + }; + + let query = ` + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(notIntersectingLeftTriangle)}' + ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id, 'notIntersectingLeftTriangle' AS name + UNION + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(notIntersectingRightTriangle)}' + ), 4326), 3857), 2, 'notIntersectingRightTriangle' + UNION + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(intersectingTriangle)}' + ), 4326), 3857), 3, 'intersectingTriangle' + ` + + var mapConfig = { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + dataviews: { + val_formula: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: "name", + aggregation: "count", + } + } + }, + analyses: [ + { + "id": "a0", + "type": "source", + "params": { + "query": query + } + } + ] + }; + + this.testClient = new TestClient(mapConfig, 1234); + var params = { + bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' + }; + this.testClient.getDataview('val_formula', params, function(err, dataview) { + assert.equal(dataview.categories.length, 1); + assert.equal(dataview.categories[0].category, 'intersectingTriangle') + done(); + }); + }); + }); }); From fab7832deedb322d68c6270abc136c45a64cfae7 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 18:16:53 +0200 Subject: [PATCH 365/402] added ascii art for polygons count test --- test/acceptance/widgets/regressions.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index f3ab3aa8..1527c8ae 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -220,6 +220,16 @@ describe('widgets-regressions', function() { it('should not count the polygons outside the bounding box', function(done) { + + // $ % $ = not intersecting left triangle + // $$ **VVVVV** %% % = not intersecting right triangle + // $$$ *VVVVV* %%% * = intersecting triangle + // $$$$ ***** %%%% V = bounding box + // $$$$$ *** %%%%% + // $$$$$$ * %%%%%% + // $$$$$$$ %%%%%%% + // $$$$$$$$ %%%%%%%% + var notIntersectingLeftTriangle = { type: "Polygon", coordinates:[[ From 490adbce4bf316c19e4acd129a2cb9d4eefc5fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 10 Aug 2017 18:19:20 +0200 Subject: [PATCH 366/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2a7fdc46..d82fa076 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.1 +Released 2017-mm-dd + + ## 3.12.0 Released 2017-08-10 diff --git a/package.json b/package.json index f482fffb..9ca7ab08 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.0", + "version": "3.12.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 92d33bf7fd7a94530e39bdd1b7cc5f6eacad8236 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Aug 2017 18:20:15 +0200 Subject: [PATCH 367/402] linter details for polygons count test --- test/acceptance/widgets/regressions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 1527c8ae..1c2e8f45 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -275,7 +275,7 @@ describe('widgets-regressions', function() { ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( '${JSON.stringify(intersectingTriangle)}' ), 4326), 3857), 3, 'intersectingTriangle' - ` + `; var mapConfig = { version: '1.5.0', @@ -320,7 +320,7 @@ describe('widgets-regressions', function() { }; this.testClient.getDataview('val_formula', params, function(err, dataview) { assert.equal(dataview.categories.length, 1); - assert.equal(dataview.categories[0].category, 'intersectingTriangle') + assert.equal(dataview.categories[0].category, 'intersectingTriangle'); done(); }); }); From d0f5ebd7ab8f3700e9a314bf21273879f3f0c38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 Aug 2017 17:55:55 +0200 Subject: [PATCH 368/402] Restore statsClient after performing test --- test/unit/cartodb/ported/tile_stats.test.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit/cartodb/ported/tile_stats.test.js b/test/unit/cartodb/ported/tile_stats.test.js index 45b1fa29..b71f1384 100644 --- a/test/unit/cartodb/ported/tile_stats.test.js +++ b/test/unit/cartodb/ported/tile_stats.test.js @@ -6,10 +6,13 @@ var LayergroupController = require('../../../../lib/cartodb/controllers/layergro describe('tile stats', function() { - after(function() { - global.statsClient = null; + beforeEach(function () { + this.statsClient = global.statsClient; }); + afterEach(function() { + global.statsClient = this.statsClient; + }); it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() { var expectedCalls = 2, // it will call increment once for the general error From 7be74d6ce1534f00a6bced6428902a7bf02c67ea Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Sun, 13 Aug 2017 17:40:06 +0200 Subject: [PATCH 369/402] Upgrades cartodb-psql, windshaft, and camshaft --- NEWS.md | 3 + package.json | 6 +- yarn.lock | 225 +++++++++++++++++++++++++-------------------------- 3 files changed, 118 insertions(+), 116 deletions(-) diff --git a/NEWS.md b/NEWS.md index d82fa076..114d4b43 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,9 @@ ## 3.12.1 Released 2017-mm-dd + - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1). + - Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1). + - Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7). ## 3.12.0 diff --git a/package.json b/package.json index 9ca7ab08..421fa75c 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.6", - "cartodb-psql": "0.8.0", + "camshaft": "0.55.7", + "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", "debug": "~2.2.0", @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.3.0", + "windshaft": "3.3.1", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index bf5e11e4..dc627c40 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -109,9 +109,9 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -balanced-match@^0.4.1: - version "0.4.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -120,8 +120,8 @@ bcrypt-pbkdf@^1.0.0: tweetnacl "^0.14.3" bindings@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.2.1.tgz#14ad6113812d2d37d72e67b4cacb4bb726505f11" + version "1.3.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7" block-stream@*: version "0.0.9" @@ -150,11 +150,11 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.0.0: - version "1.1.7" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" +brace-expansion@^1.1.7: + version "1.1.8" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" dependencies: - balanced-match "^0.4.1" + balanced-match "^1.0.0" concat-map "0.0.1" browser-stdout@1.3.0: @@ -194,18 +194,18 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.6: - version "0.55.6" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.6.tgz#11af28051c3b911fb023ae1cafb165bbd040f174" +camshaft@0.55.7: + version "0.55.7" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.7.tgz#6f09a0e9618a576ce89946e259189d6adf0c9b51" dependencies: async "^1.5.2" bunyan "1.8.1" - cartodb-psql "0.8.0" + cartodb-psql "^0.10.1" debug "^2.2.0" dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@0.16.3: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -245,21 +245,12 @@ cartocolor@4.0.0: dependencies: colorbrewer "1.0.0" -cartodb-psql@0.8.0: - version "0.8.0" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" +cartodb-psql@0.10.1, cartodb-psql@^0.10.1: + version "0.10.1" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.1.tgz#0ac947e62fe10b27916df6b7ba6c461953fe3a23" dependencies: debug "~2.2.0" - pg cartodb/node-postgres#6.1.2-cdb1 - step "~0.0.6" - underscore "~1.6.0" - -cartodb-psql@0.9.0: - version "0.9.0" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.9.0.tgz#887ba006fb499ab789c15d10f34f6afbe1f2f68d" - dependencies: - debug "~2.2.0" - pg cartodb/node-postgres#6.1.2-cdb1 + pg cartodb/node-postgres#6.1.6-cdb1 underscore "~1.6.0" cartodb-query-tables@0.2.0: @@ -388,7 +379,7 @@ cookie@0.1.5: version "0.1.5" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.1.5.tgz#6ab9948a4b1ae21952cd2588530a4722d4044d7c" -core-util-is@~1.0.0: +core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -425,10 +416,10 @@ debug@2.6.0: ms "0.7.2" debug@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" + version "1.0.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" dependencies: - ms "0.6.2" + ms "2.0.0" decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" @@ -461,8 +452,8 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" depd@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.0.tgz#e1bd82c6aab6ced965b97b88b17ed3e528ca18c3" + version "1.1.1" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" destroy@~1.0.4: version "1.0.4" @@ -561,9 +552,9 @@ esprima@2.7.x, esprima@^2.7.1: version "2.7.3" resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" -esprima@^3.1.1: - version "3.1.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" +esprima@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" estraverse@^1.9.1: version "1.9.3" @@ -615,9 +606,9 @@ extend@~3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" -extsprintf@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.0.2.tgz#e1080e0658e300b06294990cc70e1502235fd550" +extsprintf@1.3.0, extsprintf@^1.2.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" fast-levenshtein@~2.0.4: version "2.0.6" @@ -716,7 +707,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: +generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -728,6 +719,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" +generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.6" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -796,8 +791,8 @@ growl@1.9.2: resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: - version "4.0.8" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" + version "4.0.10" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -860,8 +855,8 @@ hoek@2.x.x: resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" hosted-git-info@^2.1.4: - version "2.4.2" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.2.tgz#0076b9f46a270506ddbaaea56496897460612a67" + version "2.5.0" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" htmlparser2@3.8.x: version "3.8.3" @@ -995,30 +990,28 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" -jodid25519@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" - dependencies: - jsbn "~0.1.0" - js-base64@^2.1.9: version "2.1.9" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" +js-string-escape@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" + js-yaml@3.x, js-yaml@^3.4.6: - version "3.8.4" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.4.tgz#520b4564f86573ba96662af85a8cafa7b4b5a6f6" + version "3.9.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.9.1.tgz#08775cebdfdd359209f0d2acd383c8f86a6904a0" dependencies: argparse "^1.0.7" - esprima "^3.1.1" + esprima "^4.0.0" jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" jshint@~2.9.4: - version "2.9.4" - resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.4.tgz#5e3ba97848d5290273db514aee47fe24cf592934" + version "2.9.5" + resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.5.tgz#1e7252915ce681b40827ee14248c46d34e9aa62c" dependencies: cli "~1.0.0" console-browserify "1.1.x" @@ -1056,13 +1049,13 @@ jsonpointer@^4.0.0: resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-4.0.1.tgz#4fd92cb34e0e9db3c89c8622ecf51f9b978c6cb9" jsprim@^1.2.2: - version "1.4.0" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.0.tgz#a3b87e40298d8c380552d8cc7628a0bb95a22918" + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" dependencies: assert-plus "1.0.0" - extsprintf "1.0.2" + extsprintf "1.3.0" json-schema "0.2.3" - verror "1.3.6" + verror "1.10.0" kind-of@^3.0.2: version "3.2.2" @@ -1238,15 +1231,15 @@ millstone@0.6.17: underscore "~1.6.0" zipfile "~0.5.5" -mime-db@~1.27.0: - version "1.27.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1" +mime-db@~1.29.0: + version "1.29.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.29.0.tgz#48d26d235589651704ac5916ca06001914266878" mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: - version "2.1.15" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed" + version "2.1.16" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.16.tgz#2b858a52e5ecd516db897ac2be87487830698e23" dependencies: - mime-db "~1.27.0" + mime-db "~1.29.0" mime@1.3.4, mime@~1.3.4: version "1.3.4" @@ -1257,10 +1250,10 @@ mime@~1.2.11: resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: - brace-expansion "^1.0.0" + brace-expansion "^1.1.7" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1281,8 +1274,8 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi minimist "0.0.8" mocha@~3.4.1: - version "3.4.1" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.1.tgz#a3802b4aa381934cacb38de70cf771621da8f9af" + version "3.4.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" dependencies: browser-stdout "1.3.0" commander "2.9.0" @@ -1300,10 +1293,6 @@ moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" -ms@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.6.2.tgz#d89c2124c6fdc1353d65a8b77bf1aac4b193708c" - ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" @@ -1312,6 +1301,10 @@ ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1383,8 +1376,8 @@ nopt@^4.0.1: osenv "^0.1.4" normalize-package-data@^2.3.2: - version "2.3.8" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.8.tgz#d819eda2a9dedbd1ffa563ea4071d936782295bb" + version "2.4.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" dependencies: hosted-git-info "^2.1.4" is-builtin-module "^1.0.0" @@ -1408,10 +1401,14 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1512,27 +1509,28 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.7.1" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" + version "1.8.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" dependencies: generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: - version "1.11.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.11.0.tgz#aae91a82d952b633bb88d006350a166daaf6ea90" + version "1.12.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" dependencies: ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" - postgres-interval "~1.0.0" + postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": - version "6.1.2" - resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" +"pg@github:cartodb/node-postgres#6.1.6-cdb1": + version "6.1.6" + resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: buffer-writer "1.0.1" + js-string-escape "1.0.1" packet-reader "0.2.0" pg-connection-string "0.1.3" pg-pool "1.*" @@ -1541,8 +1539,8 @@ pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1 semver "4.3.2" pgpass@1.x: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.1.tgz#0de8b5bef993295d90a7e17d976f568dcd25d49f" + version "1.0.2" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" dependencies: split "^1.0.0" @@ -1605,9 +1603,9 @@ postgres-date@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" -postgres-interval@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.0.2.tgz#7261438d862b412921c6fdb7617668424b73a6ed" +postgres-interval@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.1.tgz#acdb0f897b4b1c6e496d9d4e0a853e1c428f06f0" dependencies: xtend "^4.0.0" @@ -1840,8 +1838,8 @@ safe-json-stringify@~1: resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" "semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" semver@4.3.2: version "4.3.2" @@ -1962,8 +1960,8 @@ sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphe resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" split@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/split/-/split-1.0.0.tgz#c4395ce683abcd254bc28fe1dabb6e5c27dcffae" + version "1.0.1" + resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" dependencies: through "2" @@ -1985,8 +1983,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" + version "1.13.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -1995,7 +1993,6 @@ sshpk@^1.7.0: optionalDependencies: bcrypt-pbkdf "^1.0.0" ecc-jsbn "~0.1.1" - jodid25519 "^1.0.0" jsbn "~0.1.0" tweetnacl "~0.14.0" @@ -2197,8 +2194,8 @@ type-is@~1.6.10, type-is@~1.6.6: mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.26" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.26.tgz#3a1db8ae0a0aba7f92e1ddadadbd0293d549f90e" + version "2.8.29" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -2238,8 +2235,8 @@ utils-merge@1.0.0: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" uuid@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" validate-npm-package-license@^3.0.1: version "3.0.1" @@ -2252,19 +2249,21 @@ vary@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/vary/-/vary-1.0.1.tgz#99e4981566a286118dfb2b817357df7993376d10" -verror@1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.3.6.tgz#cff5df12946d297d2baaefaa2689e25be01c005c" +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" dependencies: - extsprintf "1.0.2" + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" which-module@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" which@^1.1.1: - version "1.2.14" - resolved "https://registry.yarnpkg.com/which/-/which-1.2.14.tgz#9a87c4378f03e827cecaf1acdf56c736c01c14e5" + version "1.3.0" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" dependencies: isexe "^2.0.0" @@ -2282,14 +2281,14 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.0.tgz#bf8ef35f9a2bb126934531b0020f5e8f96b37dfa" +windshaft@3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.1.tgz#b5557fa6b0cfa13920904f57206b86f7aa054f74" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.9.0" + cartodb-psql "^0.10.1" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" From fef0dc302a6a534c009062f2f92c9a425b7f5d18 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Sun, 13 Aug 2017 17:55:45 +0200 Subject: [PATCH 370/402] Release 3.12.1 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 114d4b43..5f3fd700 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.1 -Released 2017-mm-dd +Released 2017-08-13 - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1). - Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1). - Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7). From 339f1aafa907d9f83126cbd5829b849468141731 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Sun, 13 Aug 2017 17:56:23 +0200 Subject: [PATCH 371/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5f3fd700..2a392e06 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.2 +Released 2017-mm-dd + + ## 3.12.1 Released 2017-08-13 - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1). diff --git a/package.json b/package.json index 421fa75c..8b3f1ae1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.1", + "version": "3.12.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 175d070f09b412179f883a20739cfbae7cf6e40d Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 16 Aug 2017 10:07:27 +0200 Subject: [PATCH 372/402] using const instead of let and var and adding assert.ifError --- test/acceptance/widgets/regressions.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 1c2e8f45..74f4544c 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -230,7 +230,7 @@ describe('widgets-regressions', function() { // $$$$$$$ %%%%%%% // $$$$$$$$ %%%%%%%% - var notIntersectingLeftTriangle = { + const notIntersectingLeftTriangle = { type: "Polygon", coordinates:[[ [-161.015625,69.28725695167886], @@ -240,7 +240,7 @@ describe('widgets-regressions', function() { ]] }; - var notIntersectingRightTriangle = { + const notIntersectingRightTriangle = { type: "Polygon", coordinates: [[ [-29.179687499999996,-7.01366792756663], @@ -250,7 +250,7 @@ describe('widgets-regressions', function() { ]] }; - var intersectingTriangle = { + const intersectingTriangle = { type: "Polygon", coordinates:[[ [-117.42187500000001,68.13885164925573], @@ -260,7 +260,7 @@ describe('widgets-regressions', function() { ]] }; - let query = ` + const query = ` SELECT ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( '${JSON.stringify(notIntersectingLeftTriangle)}' @@ -277,7 +277,7 @@ describe('widgets-regressions', function() { ), 4326), 3857), 3, 'intersectingTriangle' `; - var mapConfig = { + const mapConfig = { version: '1.5.0', layers: [ { @@ -315,10 +315,11 @@ describe('widgets-regressions', function() { }; this.testClient = new TestClient(mapConfig, 1234); - var params = { + const params = { bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' }; this.testClient.getDataview('val_formula', params, function(err, dataview) { + assert.ifError(err); assert.equal(dataview.categories.length, 1); assert.equal(dataview.categories[0].category, 'intersectingTriangle'); done(); From 3a22adf966751ab1ad5815233489635adf8d473d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 16 Aug 2017 15:35:34 +0200 Subject: [PATCH 373/402] Update news --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 2a392e06..436945cc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.12.2 Released 2017-mm-dd +Bug fixes: + - Polygon count problems #725. + ## 3.12.1 Released 2017-08-13 From f3b7a857f2c507e6d575c2fbc45f7b4585eb4a93 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 16 Aug 2017 15:36:16 +0200 Subject: [PATCH 374/402] Release 3.12.2 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 436945cc..982f164f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.2 -Released 2017-mm-dd +Released 2017-08-16 Bug fixes: - Polygon count problems #725. From 0ed46930ddf7d5d01d04210c125a8e487e1dbd39 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 16 Aug 2017 15:37:29 +0200 Subject: [PATCH 375/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 982f164f..46655deb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.3 +Released 2017-mm-dd + + ## 3.12.2 Released 2017-08-16 diff --git a/package.json b/package.json index 8b3f1ae1..3ae6fbb4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.2", + "version": "3.12.3", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 4491fa2fafa172e2e497cb59cd546da3f7a3815f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Aug 2017 14:17:17 +0200 Subject: [PATCH 376/402] Upgrade camshaft to 0.55.8 --- NEWS.md | 2 ++ package.json | 2 +- yarn.lock | 22 +++++++++++----------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/NEWS.md b/NEWS.md index 46655deb..ca885912 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.12.3 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8). ## 3.12.2 Released 2017-08-16 diff --git a/package.json b/package.json index 3ae6fbb4..26cc7847 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.7", + "camshaft": "0.55.8", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index dc627c40..f47cd66f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.7: - version "0.55.7" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.7.tgz#6f09a0e9618a576ce89946e259189d6adf0c9b51" +camshaft@0.55.8: + version "0.55.8" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.8.tgz#a289ee02598a764144d5a56fcceae964b425d63c" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -205,7 +205,7 @@ camshaft@0.55.7: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1525,7 +1525,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -2108,7 +2108,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2116,7 +2116,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: From 7aefca3f82656eb6a48e62999997de5b398dba67 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 22 Aug 2017 16:26:36 +0200 Subject: [PATCH 377/402] updating NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ca885912..66f95e6f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.3 -Released 2017-mm-dd +Released 2017-08-22 Announcements: - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8). From cd27d6aa02422acb1fff125846df52f3c6d64b86 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 22 Aug 2017 16:37:23 +0200 Subject: [PATCH 378/402] Stub NEWS/package for next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 66f95e6f..1d1c2caa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.4 +Released 2017-mm-dd + + ## 3.12.3 Released 2017-08-22 diff --git a/package.json b/package.json index 26cc7847..a4bddced 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.3", + "version": "3.12.4", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From ca0b927f517ea59baa7327748a3fdf9fe0154dc5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 23 Aug 2017 16:34:42 +0200 Subject: [PATCH 379/402] Update camshaft to 0.56.0 (#731) --- NEWS.md | 4 +++- package.json | 2 +- yarn.lock | 58 ++++++++++++++++++++++++++-------------------------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1d1c2caa..828b4081 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,10 @@ # Changelog ## 3.12.4 -Released 2017-mm-dd +Released 2017-08-23 +Announcements: + - Upgrades camshaft to [0.56.0](https://github.com/CartoDB/camshaft/releases/tag/0.56.0). ## 3.12.3 Released 2017-08-22 diff --git a/package.json b/package.json index a4bddced..462d9bdd 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.8", + "camshaft": "0.56.0", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index f47cd66f..8876dea5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.8: - version "0.55.8" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.8.tgz#a289ee02598a764144d5a56fcceae964b425d63c" +camshaft@0.56.0: + version "0.56.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.56.0.tgz#6adc8ae8e5b63638551ccca97cba8273642e4f1a" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -1747,7 +1747,32 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.81.0: +request@2.x, request@^2.55.0, request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1774,31 +1799,6 @@ request@2.x, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" From ed0bfa5f63c35807a3917864bde07f7ea60c2248 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 23 Aug 2017 16:53:01 +0200 Subject: [PATCH 380/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 828b4081..3789a8ea 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.5 +Released 2017-mm-dd + + ## 3.12.4 Released 2017-08-23 diff --git a/package.json b/package.json index 462d9bdd..d5da7a77 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.4", + "version": "3.12.5", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From d644376f88d6c4d0e23ed24638d0089b1e242c10 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 24 Aug 2017 12:07:43 +0200 Subject: [PATCH 381/402] Update to camshaft 0.57.0 (#733) * Update to camshaft 0.57.0 * Yarn.lock with update to camshaft 0.57.0 --- NEWS.md | 4 +++- package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3789a8ea..5f59077f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,9 @@ # Changelog ## 3.12.5 -Released 2017-mm-dd +Released 2017-08-24 + + - Upgrades camshaft to [0.57.0](https://github.com/CartoDB/camshaft/releases/tag/0.57.0). ## 3.12.4 diff --git a/package.json b/package.json index d5da7a77..78283248 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.56.0", + "camshaft": "0.57.0", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index 8876dea5..e9c60d7c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.56.0: - version "0.56.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.56.0.tgz#6adc8ae8e5b63638551ccca97cba8273642e4f1a" +camshaft@0.57.0: + version "0.57.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.57.0.tgz#22c2ab92903e43d5af6b763870839fbcfe82c650" dependencies: async "^1.5.2" bunyan "1.8.1" From 03dc2601048d248eed79912ba04c4d114790c6a3 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 24 Aug 2017 12:09:51 +0200 Subject: [PATCH 382/402] Stubs new version --- NEWS.md | 5 +++++ package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5f59077f..79a2bdd4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,10 @@ # Changelog +## 3.12.6 +Released 2017-mm-dd + + + ## 3.12.5 Released 2017-08-24 diff --git a/package.json b/package.json index 78283248..0a5d3621 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.5", + "version": "3.12.6", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From fc0dbaaab12b0bbe377eafd6c9ce06440a55467d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 29 Aug 2017 13:04:20 +0000 Subject: [PATCH 383/402] Going red: Bounding box parameter ignored in static named maps --- test/acceptance/named_maps_static_view.js | 26 +++++++++++++++++- ...ed_places_simple_reduced-override-bbox.png | Bin 0 -> 95961 bytes 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/previews/populated_places_simple_reduced-override-bbox.png diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index c7ddff2f..9a216451 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -192,7 +192,31 @@ describe('named maps static view', function() { } getStaticMap({ zoom: 3 }, function(err, img) { assert.ok(!err); - img.save('/tmp/static.png'); + assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done); + }); + }); + }); + + it('should return override bbox', function (done) { + var view = { + bounds: { + west: 0, + south: 0, + east: 45, + north: 45 + }, + zoom: 4, + center: { + lng: 40, + lat: 20 + } + }; + templateMaps.addTemplate(username, createTemplate(view), function (err) { + if (err) { + return done(err); + } + getStaticMap({ bbox: '0,45,90,45' }, function(err, img) { + assert.ok(!err); assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done); }); }); diff --git a/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png b/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png new file mode 100644 index 0000000000000000000000000000000000000000..f7fee54c020fb306e1a9b82b3a509c4e670a6d86 GIT binary patch literal 95961 zcmX6^Wmr_*7N$c|5E!~ax_fBpZfTM3h8em+>F#bMq`SLYx`z;H1f=iq-TiZZ%-MUL z^Q`sO{-&xdgYk~&9SjT%hMeptbr=}902mn9Z4@NvFFdpB5Ez&M7`ad4nxL$cPVYF< zMfdaW<8bamk;5oyS7|m2ZrO}kSt4O8(vK`p6^$Au!F!%HChjJRF&CbG|8*1AEu0oQ*za$cK6ENPt)VDmzaTcwgn$3EB2t zefN>>e08KV91aiP?@n2OK!8(?O>}o#P3nH+!pPlwj^M09JpML1&x{<=(|>;8pC?^{ z{EEcP*O0(?o;@9cLSL0=u)x49U zSuW^2h5fbg(!1-4JPJ1l0NcZGSt;kSBQ-H zU41gYz1VHW*=X<|$SIx@01opM3x7r9V8qmND$&yF{5a8{D4mv`+k1Uc>wZA@8nV)@ z|2PGDcyPJzmve=D1IBE{EVJu(IeL{&Wihdp#d<*xgAZh?wq46R+0sk(*OuNf?Ch*7 zI`M0yX8-PPbkh?s^l0Q^OiEIMHdJH8T$4_B;CkTsYhayu;nNE#v7?lESjP`&+1!|q zK{%H*vnkq;f;u5hnlL7p23GW^MQ5<7xl}bvdoE_E$@@)(jjZpM9r}D?n3(p+{C391 zCLZJcSV=+uv`-@_Rbm!dv_t%tmduP-p52^|Xey5qBd4?3*!=uA^3Uh#V-t+4t42x9 z@22xVN9jPOLBE_`8MEXYDIXM9eBGSVl*{@rxd;O+Cz374&a8fj!;P6oLKa(Q&Y`s{ zg}xQ0_E=0z(ujwOiuF3M*zFvh?Nzf)|32{dJOUt4l{6x6+vnd^OGjfdEAqm*sIJEr z2}ucQ%z5gt+mLbp+9Ewx(#AAr0DYXTsBKP%7{KQ-57=2jWvt;}k-EW$jEo$PIfRrx z2BC)t*&j(k4~VGIavghoRe3n7y7FvCKatIlNZonmx7tURZnabX2N&;t4)ys8CfYxe zOPZAAzg738fRVtiYFwqOp7GO|5vyYsw+z|1!gK}k&jv;r$D_n>*{oKL4JM+e5zm{^ zU@79D9~D;|9dTIILfj;i=l}HM4?UdzTSrEqaYmPY@%IPcvy45$S5ci_c&$JAsGOwc z%8GCDy8+w{CV`=rq$I0^krtz1GnM%;ulq3WKh#D%j3`&;w9Om- zMe%bsN)miEiLG3)9&xw5`S4b!lrPJP^|CgfxfQ1HEM-Z82)Q8qW@w7i%Z&<<*NWKK zL!SrFm8{I^ix zn2xONv8Qm{FiMd#w{1Y(f!5gbRNoTU>vXCq=)>udydiFxHd0CNHOP%MJ^bHZlpw6+ zA$|C!3J>ULh>T6jRktPCU%LOJCf#vYB zIHRg~wC;bErjGnk+dH~?4|XqL%(Zv-zZ&?Y#Mj(-h#sg07hD-P4L&X*|Jf~COJ*yL%`_8XBZ6T+L zha)QPlu}UxM|TAnu}MkWV6wAhC1Q*5;5FoZR3f7>FqgUekH>hv1+IJgd1;b<{j1b8 z8wG50-7R`=-l-JfM4r~n)j#HUqrLfS+qFLyz_86fzHMHcm7T=k!=Q%+Se;6gWy7kl zyn8Ij#P3Ul4vG6>ErHq-%qBFtN3-+IQ8UK`z0bJ`A)q{kt zlkeaG=Y=^2(0&|=V2fd^&c8kldcH1%@cdhXcE#TOAMET3(ibQ{Bf{u2!xwjJZM!;v z+R5`{XQnyZUa{4qhUfYeY{;Mf*au%V?9DZbYU}KG9r0r{v%nj+>&k^vs!Z{MZ#n3O z7+u`g@9d-AM=1~xj$kpM_l%F7*#=tKUEI?1)0`P?#L>1lo+u0QyAdBN9$7Z>U(DrbbOb0%`#vcSR=qmJ+hV zk82n~ettl$sj6Sbuei+f(cD!)KzmzhmS(d_L=nwKy(ui2imY9lLl5!bb40res7hO3 zv#R%~M*$#PoktxJIQRXfnK~}B)h@oHf;iY|LE2&zRUpe@*>T>2)rCY1IuLME3LYFe zAr3@|TX&Oj(fQ}C*iPY+EZC{Ol;X?;HRrQD_kJtBcJ$cl1HmCn2r6VICi0OxY6;e6 zpCO2uGY^~PpNeWyHCEPSklzfJc1ol;>LwG#-9wT#qC>o( z?^YZP?Lr!+#5)HX$dKW&fUbg*E_cQgRhK+CZHA- z0J8HItxy^}oI!VhRi`dX6p6+&;d#Ul%-xw;Rq4$$eeJRHz0K&2!Yjf2zjcqx-}o!2 z$SL-8ET2ww*JRb5fSPafwIHAwbEAuYZ#n@jEHJ#6FC>k-qwJbsEK=MZ8yJDBLFhY$ z?REr+3;41-(g&|;>Sw*;dnyJ>nwhtzQP+|q!Rv*7V##83@NB=*Y_e5*sj;H|jy=x8 z4`j-$aZ{+6e@spS;9KyWC(eyva^uWGBF9vFSvIv(37`Pb?G2VW zrO|z!r6<7c`y}=uIP`YMZ-BBYhY=xq#6Zm z@8!ZfnL@XNTk-o#^66y%{nFg~HKpq+=%5rbZJ;XcH&I9!%=@bg{V4E3+#F8uU1T{T zHrWSBlSiG7fkyfdj#?>le-Le7G($KwSdvNXwnpwpOy!quKj@R8&n`8YLKi@-HBG#9 znIpL$u6xWtgu<_QHdY1Jn6p%G@1N4lOu zkQZHVe%>U$(%s3b($H3Nl##LCGG6nk*6}&X=^BsvDg)P^6gC~}FT)k)kk39-oGuvxSjShhQC&{$ z?!`~lxcx_FY%+~Fml?a02HMtVIV&*v2Q*dE=PQS2ef z&un%=2vvlO1K%Ix%~4;Nzbo`2KVXKFM$>`q?``m zy9=u@1>o)|O;KkF;}#p`aVD2+C2~O?NngPr2xVR5o`en1wh_g*=0rr0{~w^+ zqV{Q(`}Kq@kqC2&r~p zcZw9-LSGpfu*i7g#e$p(3rCD83f$H!B+~i>VWZcvOWpKR8gJXjTHu8ftD)QgMfE*I zNl8A0U8rXs^oWYe8)@X)KURJ@kW1V}&axd1$K2nZ@#s+!J>!#EU+n*g?YBh)_SRDB z(ln!2Tv?n7UQ+`!>8dW4d^KJnb$5b!16%W;4UXlHbThZh+!R>p)wHD{ve4e9j7$qB z>YVgM65gq>UI#z7}+b_M7EQ8x#9V02L*ku@@ImVyO3qDl()lwnz3&2AQCnS<%+mA6f#CJ&HsFvetb!a z5Y$aC#T*fdV&y@7u>(tW{Xnl_E8;E+^z`WlcCiKeC<=LYbsRA&xCtLG^z*VvWWIwW z3qmJV9RvOCxM?L5cO#+i?nf07O^@!mo-laOsH#JP6((jCvX^vQom~iyuVEJrw?qsU zJ@#UKh=>q|sg}TLRO^*`lDFqx+7qwBesnLHzy{>Lm;V!}E#i-}6?s|o02yhi>s?t# z*LosW>c%v%H!ME-P}d=8<{Q69f;g!|ga?7U;XH<9PL?n9bS+>pGC^?6)4bK= z3;|K3YS#M^I`Qb_wehnrIjUh>sjR~4eq>WVbO~Ztb0f?G`6n*|ifjmi0mp_yipba- zi*=8O+cO`g(wI~-(J>%Oae!}+BJL_mTcww++Q#}!PKrt|dIS~4#Z_5fnT|Tl#cl~e z*7JyD>P)jmgi)C0C@&vaEmlZBZpUk!kyMEv&BI;siJr&}_{ zFwM7r%2S+ju<hMdyTuCvqig4?cKZP)CAQ%o zvxeZ`l`qph*6T+fYfBJ(H7}h$GhRIuzLTh0+h%~*(GMr7*)~%;V{I>l0?>IBC-t~3 zv7UrV#XeVn6lgk$17yDPY>Sjc$3;j~)_zArLxNq+4en_%_Nj&f>UYZ^)z+7!-EsFK zW4GZ?S>EyVLhsstS_cSlL5lv!B+@cjjT|!9j2I_vEIA-g@0Qf|bIBCi+ZvLw{0DgF zkymNLRajG&hJXU?C(0IsX1VLc$wc3H|MAIfksl{d<{B|}&-vzdU$<%->HPRYjR&I> zIS|7=ETwS)WozOr|LfvIlxvb|rTj$*`f&tu9-75DBon3DwZ za}uq6{@wU%h>q8m4}=UeoR>t6G zV5K*$hwm7zXj=HoU%>o&^fD8w`bGW2MX&@q^Z5c573PZu`quqJvgQV?k3-Tf>5z6b z|AtQ)e`Kp>X>kMO2@XIJ^CkQHoEd^5*N|xY7+4PcNlqZ60lesa_pyHk`Z8w7FUhDo zS6wMqn=6tdl$?T9xw-788g((NrJ+9|`OqV~e#$ONosK5c<9e7^fVkg0h8J~QE;3?% z$rkt>gm^V)&$Ms9+8DXNG(4F?&RX8aYzpm#*z7@yXb3}7pQIc{yfAq#Unm$pgsSnX zCEH$gl+dVp*=R|1DP4CemgB>>B!Com9xq)pP{jDEy@U0D@bEcdu)e>5;3lopECvlJ6&$5K70L#cQ87Jev6Ny%uMZhd z!EW*I9Sc`R8*9qP_{L7?$=oZ}5t79&(B(0wO|An@xC}+JEN^CMm5f86$62GO_rnj0 zwX|+wxT|!irb&%PN1H5`{l<|lwkL`!opbRT8CvHtHi~hP zd6RXxiK*(|tA}Ooaf%L70}HQ*zLI>uMiAs5D9``~NbO93ucCfGj5g(-k51J+9z%6G z5+Wo?gB0=s2OLs$vKq|h$T6%p#bjuu&NH6gf7Om2)tD|mAvg^CdLq4!chE%G%q(b* zr^~`XhFJO{b!6E!tS(rR*9{K&r|k9$H(`J(Y*GKAJaV{oY@^nFwes#xJ ztdk zxU&w=*&?yl>5cTR9){kb7*hc|p|WlPQ!4L3N9|VDe%Q9F8i6Velze+>l#yWV9A~Wm zr(~~^+E9aRgtXwSY0H8Nc#~pJGU(Z6RB*O7`$UrvfcBlNOaJOj% zm6Wwe>6q~ao@Sp+eY^u@InWg`SfJ~&3l&6rN?=zM^!FR4 z-O3U#ZP8(|+_a*nc^0C047qN2Ehpmk#?R~k*GbNNx{1POdbn+e^sk3^JJV`f)CSe8 z>)4>18yJOt=-eWMwOXZ8@km-BfU-K|aMqsHigoV|Xh}f5)Fd7U24Ha(-ovDgZVv>? zg`$T_%HSO|^E)mXw<2Wh*bOimM6?i<>?wCkM%XJPh+(=k$GC8vsLhb3CCT~;L&yBo zx!#RnitppjiBm>3a}oK<@tmrN_(HX$W^5ClTj(vJfAkV*@Dr< zIFiRe%Mrh(#YnqH?2x5!*jftg{oz~xh{Ffi(bboEi`T54%I(ALtQ0p92h+B;wG zLpE;u%{Y>|d1@v!{_bE&jiX$KOgnCk8bju8ZE|w9_HMd~EQaAPMQPQR*8zp*%U@fl zWQ>1`8GUb${gevT@t{iAa@B=0@k}B_mtqmVrRpR`kiQ6#M2XZg9_e)?=~Hbb)uF$M zqUdi8YpHk3dTPHL*ze-; zRa@pnW~}TE_zC2-r_z)wH*l!57Y0s@FGlXCI`icq#Nj^d)6c=Ol>|$YqZd;|J=cmKq6<(8+lHnxAa%nF#fP1@fYn{q(-to*k`a-)d-e>pA8d#ynw2om?w~* zp3*vA&Q|KC<$CxVu-zNyAFskhT2sT&BWdhJ4yKZB{yahgp)e+>$+_u72}g#kYNU8% z{*8~J$Xq7AUVCRMJNwKdNb#}~St=QLQ=yo@-m6{dX}X})6nS}`Gwx>1lL{7yp1!qa z8y~-d6F(0)#XFZJA8>gvjpi`{xiu?XKxxZKj0arYH06YCTP_V5RNr(6mr;+<8#v30 z)+8zSA%ZWo9dTs2eAyk8uo9eD33hFU(ifRT<^@a9;L1-66m8;!pCU4Bm9JSBc^^cp z#H9%45zNPFC%$td)kt1bFfIO_Q1Q(7v@PS=FJ#``8GOlEyt;)N4QPV}CojdYK3@=a z6NeCW?C@>ha=K^b|{8eaEO8*r5nx(>K!=T(&A-cx&YEZEdeSorUx}^pi*v zgXnA-E7JOY3{U1{z`VI5nRJm{Nc*k8M@yU@zkj5sY~g1Civ=MDp+v-%sT;cCE&+!) zDwj|lFQ`eGu(96Tyy3*pooM80v7Bc+;-jj~cyDju&*Q3$gn$r-1q4r(%|od!1VY^T zukBE=lu;wu)`;CV1%f#;2)<1;e8!^=_hv|MVgHi-;O`lDX+dNO_;y^_4{Zh}tHB&C zp9DRby@)NxHJn!#K`Xv0wNOZ+wk${yn}HAH{!L@awk=!Ju@wdn_szi9r%zfPZhFE; z<%N6NkM}`PA;x%%9*dFsuQ`!>NF-*gC#P(kK$mfs3KF@feWJrEEYz}5;eX`gWiH0A zg3jeQq3u57cIdZG1iIB$O1`yg_w;Ge;kAeQi)px{ik6Z=GA>12o#r11TF4&W;|OwB zN82*Dn?miq7(p)(UH^CV)fZ|a+8G!4oHpA9)@i`6Y$la!@JLU}(APDk<#|M?+p3j# zXDYiFl8XkNZBLyiA{luP{3{o}1a!Jiu zhZ9Wl&ILuTdC=0O)W6DUorXX@D1sF~cvR=JJ_s5WS$PaL-Oa`7K0JrQHzWYahWU9s z+Vo2Q@2i!k;t2VLx}v5KlWXIelUZ!#z{kiKu8yXWG3N3TzF&Y;!5TJVxTpc<=Rwdw zA!6>SYvC1f!+BXe&?r9ElHVZv#Cju3&ODmh(47J~d}^RWx9)wRCDctXrGyRoAggv* zj|$uE7}&v#NE7r^|M~Bn8QHe3>wc(eBPN#fRnasRttr=KZ9sioeO|b*X68cCn_+X+ zHS`-!;(;#T;AQI2MUYlYV5G}otgrz78(GMMwSr-orBF|y*TBoosA<#~&^n5zv3BDt zqwesKYXKEs(JAwgae_x48>CBU@^Yc%=E{5NXo~~glv`{5_RTyntJFeM;^u&x##d*+ zmPoP>u^0x2O`E$8>hR76G z45y1+0xGpKn3qOHT9*z;$7kNGLe6~j$Z?06h3r8M?Ib70Rc+_=&j9nPkEkPXpT4p1 zIB%#(CaQWWcNE!NK76oO`yQH$UF)!m9-amNCCQZgE6TO!8LEM#C|4VG&c4@K z3+G)|-Ra&wvc)fGM3lUi2pQt(Lcnbr#mlKZwXg6^(13`1W~?Q#q4MgG-=hsllk{Rr z+lk8j(tXZJ2RQV;#m3dwbJN!yDGy4D0_9fzesv^@8-6F)DYL@S;Q(5)GJ9bxJc&`h z#Qfaz{T#q5{Au|x6Zh(oMkP4^VWa_03#R~2Zx`Ke)X?zOV#udW#4* zzUZ>4NuCmd&YVqL-Etmazh>!)2DCc9W_-7sygRw z+6vbxcvlufkdPD@Lw=3t*7B~L6^*cT@jfLS%25~J>hbEF1wz|jxu4^dsYqUd!MHfNe z`ALsjD1HNW#KfVJ3_(@8OEbf{j5ag;btyg_z^QrNa_i7jk+Gf*QQEot!FY9AIc$;7 zEsREF`f=L!7(Z^tC6iJvykwK-&xQ356Vu8aT4km+ID)IUkt(<6;U4B}n`n?=-XTZ` zKn{l!>w~50Rs+N0&^);qOI?`)C8|qCsYr4r;9C6TI7u6>7tWkpG8}0!^naRmxoSOy z{8or~%3E<=4q4%NxUaNtkt3udrL>=;0y$ClO9Msk{p+*j`n*ak1Me)|l^mweUD}Y> z2{Pj#Pq9MHzcHvG04H1cr(?}X!lv{mCb6-Efew~ObWe3M{ z7Jy!4-X}3L=Hy=_8Jb60A`teZP2?H=b5A$pLN=IOBe9#Q4hz)kM#6YJC5`)G4ra$Z z8zMh5c44JuF;?sEmn5?Oozej}=c=BV7?sNJ^CN{~9T$Kx@B=#1?|yvBtUT@7exYsO zAl!)I*j%ug(pt!ZLG-5Ca)^p`rr;MzNh-ATZP4<_JGvD(HLIu=I;@Qu z70(0#s+F9K#^1T%MvV>yGfo^!Q-fU7s#V*QPXWT@YVB!E(vU?g5(d%Xp7irkZ@mqI ze^PXCLP+u|f$Q(^<#=>~xo&=ct1{$9dMN62VU(4I|A6+3gM8$W55zZF7ZkHyb>iz2ZG;Fon$ZI zIDY@o=2*ObWY;;KciM~v%?2KXE*tqXFGL&Fu2!!UyOTL z;pz6mEYQA5Z7H8A;(5^F?(4#4N)WqIY?JM}DHv`YZI0KvJ%$kU9E!G&$N8eSlB>b; zPm1~0MhB!7|J?H!qKP_+2%v3XiyHky{Aj*4SCO?mQEM-(mt-)$qYi9gQ3Ul0+>gHc zPK{G0Ha33f7xNh+Y4qaQ-tF2`Xb0kp(C3Ix$^AFStEkS(N3jq5K&;A)-@Ujb{Ha>s*cQ`b4H2=4%V zm4i{gFFxkiJKHzDph&uG9VW=*SZJWnE3iIoaY%IF0rV5ZGIirR*xqK3ehX*(#(8gb zSBB`ZkQm!op)z)K-!=NL2hMWDGdkjg?&9QFld4xk9^97rXkMeo%o;6( zTE}8`c0AR;)qln6%Vk#hTL=)-mG6dvGM-49z36QPU!82bsh&#S3H8++hQ8?QCMr;5 zhGUqakJ)RG(9tuPVyR?N#v_s~YVuCT%XpSWv&oUu2txAjP`Bojtag|1wi1Rz`Jo{h zfE^Q(qOv(g69V&5n~Rkns0bx~CCmh|Ia<1vgTYD7G(49kG=auW5tNK$trkOuHT4Jr1W)u6oEbVAJocF}NC6rE9 zqVCoZUB4!ZHlOc)F;2574(1|S=}0)2rsGp#)MByQe75)_4z_^8^ij(@T<78`?lLL)7fpe^i7&GM}WZV2&lq7RVG7kj`DYAs;n9fV(f-G^_ zvKb6wd&8m>F0xH+u9kQsVR9pd=<`il`~VS81~&M-Hy2syQ!;6y*OQ!I&J=aA&Sz7F z@4XBpzDYb6gnGT#C(49#&R0T4H2=3D_RH$?OPKl>&0wprnnScrf3z1GSBxi7xBW10 za}BQNVuK88pZXnxg+Bw|pv8%0F2G(Omsilk zb)usNlrOT2>YKt?fDfgE=`jC6hwh^y(oZ?tB)`b< z5O=4vuFq$quN@xFPY{n4DzMb4!)sZTA4mRf`a6jD1CS7Pz!ufed2Xz~fgKwk4* zZr_Nak0fcm{|2I{%vqqr zyE5+>xGqye1(O*mJu{|AKs z7T5|uXLudjc3kW*xchq$sIJh+2TQ1ghbV z;x|sfkr;Azh_{o4OYn6jY)IarC_e?7U zc^SsU!u`BUIRV4|t2{v!-oU@G1v^Xt1NKYZQGVum26c5z{UYL_`4W(J+s0b@O7nH(i-3U_4cA)A4k&PPg5bDmj zXnx%u;<#tY39oxhg_*Y4p|_*ICl*%Hf(y*NsY0!X1q~U11rPn2NdObGI5srN*csD5 zjINX;GiI7AlbR1zSmJV6pUCJF4Cy|mdk#iy>Q8tRYg&o(ueq^Zyi<^KQ8M3QR;pUN zsK;^@h3ydM_jb-5E-Z`c=|||Huee%y^R>f&!$ot5%H&LCFc05B^Kd%556)}y_noT{ zUx7cJ(ELkUiP+XTNi^l4gC~{x%MU{_q^fcsU7@3^!7s7|@|b$@R3lXZ3}(vdT|iED z`??&|>Bw{1LP1ocpH@v=z@IIgVDlhIbdkFE4idepuMf)J)Zdl0O^&cq47&=$fyH*= zTI23k3iQ|u*9nF{vOmfMj*T+dtQQtqh}4Mbcz%($delB;bm43)L&Kkb7*&>QApMVt` zy)H!(P>G+(A~_92qUfXSf%TKOw2s&>Je|1>*?wPPH4)}5Yt)bsat7;as*>}9!pa3P zjU-SBt()XXZJrw1&?z`Ih@XG)X5t}-JHSn$YEn00jp1@PN`f z3AwJM)n;?Tlw2Z=KoRZuANCzJ?Iq|kZ=!#%i;9TbXj})55{bs2(et%5{8Tba8=#eV zzeZyRV=gp@D5&bqI|KX!v(W}a7DnbC={cF2YCWt(yLH0u`~LTPH`IP3jFDG>A^6=F zOGKVX$J!BtPs3`yK;A-5mW$g2gh0NiFn+GSX+mh65fQD<7j;LCk3Uw#q9?8#pMx92 z7-#4RqtisuNGEra2ep(r?r$FYmVc|Crr$Un9;_)aPlWu8T{2FdFvQIf!yX^(4~kuo zK2`|#-Q;}at*VTK_UFg1a$3p@ldUr%N!iY}E6)>%A#94cB)ai7shzdZ%L9tN16AX? zFhWcz$Di7;yj0XJ1T{7>_M&(fm@==={(ja{?w7>BecIM45{xh}4i(g0O7wLhV2PbX zAqtsw0h<(xW<&%8dYYKI_V8Q)e>KHTN7{`+y?AjAq`zi4!*mQ(zey=HJl&t3tjx8qw{=FJcQfH_!e)B8&s@*EZ3IMRV zQ0~VO2W93_bL%r8U}TbdD1@NGtAm7wpMm2rM8n5r;3jZ#qnt}zdcXN}U zPo|sG&j8FW#A6QSB5epjh=o1&`%J;+T&<8S zE>9Kl+Z$?tg2tlcGIF2xe(h(~Zb)baSp~0d=csuYUyDiN@5LjWvhz5jo9K%JN5p#x`4`-HLM*yxV+=dNlyUEc{j zW<}WwgU-)>-z*VLv3CWE3$?Y{gV~Zo!1ZR->181k%O=-Qx&5*3F(tq`JO4d5{>btB zy7H4Lb$0}1z0X~tHXAVZgXPONVl&bw>Bh0YgE*uZ}L{-&>WqZeU_aaH&96{ z;}nqwjXeytP@~h*+c2jmQuiwzXoK#0%`7cl?1+iPLd8a|(OFmZo0sYe^*?4K-bUE4 zL%DR`HSl!*55$eP+(2?zQWV>m@5sNQ>+(NL(sO23!7%;_?1^VWe+lsdCiv9hjp6o; za15*f-PVl%D$B?28X8B0)@>K%e+qEegV?dh&n)%g1`v5@erKFz#Wg=Q78D2Id)Blz z`3!G3hGSoNoslFRx?`nxA7VF+lk&nXzd4prvlDB3umSZhKeJ3kWOJgm34Qq+;vG3< zX-IeTaw1kdp~XYB3dxkcMq?Y}-3E3!mJd5_2c6D=JDENOt9>HRpQXd84goc!JC0mp z9^!T8EDqy4>_=ri)${DRsE&;i^7U8#Z)E7XvSr=?m9+Bi9lNh}=z9{~v)Um@J69P7 zKGsnPNG49i`EGr?n)I_bkA3AB$4;BR=hAuNmPE))%GTb>&>#)Bs&vxqyfnYI-g(5_ z)O(ThW(MVE$x$)heu&ie`6=Kq99IS}y2x+unfPjzmpchx!M%MTZ(r?7<@Bkm9QUxq zd8_cSd?Lxz9vC3x`WL3}$M>tbf<(>wZ}iqC-sB5KFOGAN`k>U(>+fH_41~&DE~iJi zj)vt>aPMDWDrhjG#F#6My^IogXp+_^MV1z3jDi}Llx4K zRHeJ3DD@y-lfQSEBmI*(;q0s(tw(gMvQx71aofV&H*BmE@F%1zw++XH4t5hN)${c*YuY*vdSg5~WQRD_44m(Ue{Ys5`L>;RAk zIq;N*aliT&X%tN7DykM~1z}C4Q_`y;|Kj%0<-cm7w+=wL=c7+}^tf55ANzsnOwFX% zNv@(14ga$7mTh58?;mn1R654>0A{B$ThX9T-r$B!0l2R{7evLydvySAlb zECU?r&a*iuUPs-~hZ;vmJko+z8-P}AxkAWTv5E~sP;6K|UK-gziZPzw-wQE)ceet9 z!%_mYk}yC+#YdzyrY1hbvtPVNYErOn59! z5zhg#g;q@Z&oe|#mv~(sgzr0Im8Wbr_J8TN$ z@34J)X((-C_P2r!Oiarj?&vSGsZobCJqcs2!oijD@KYhKJaaTrp_dHbvSZ*}+mwSr zubNf=fK)=5nBhUHC*Uwcp#OA&+9-k*Co9|M3(WUq;-gM1do9r0D6MLq{0z6r7W^3Rv)?QYrS*u`6FE6vM`_b4)g_4_87ZEcQUcN@zUXK|ySM z_d!X$kupppR6O8=gaAttfe%L*JGQ~s10^3%AJYktXND31lLO&P1RDb<7zrLYEkmGx z3+<1ReXBHGGkkxy=gc4_fzfxE=UhfdMW)y}OJKJ2@rB*3@5c;=@NUwVF5yeb&*JX6 zX%Xt-dHLd51Mal2H1s^4u(ytpJGzIwFx)74UHd`;hvJn!e{T{fgaj*D6AxqzP6gO9 z=c&b-_h{J-AAdN-913K8B5z7i1u;;XY3sm!$%M> zRV$JVnb#*pbJkWfiYmU_sZ*7aTqIgp0U+gjC-^V()s!Bnwu9k<;Q1Xbt@byP5fY9bAwBk zQv3k3O4s-Y%RkocoB}rsG4oI&8PJjw7Lq8r9r>~ej~8(A)h_`-tqJM`JJfS<(%8{Z z>7mj{YGGiplE{5r_gqCp+u!=_DRdlOB9`qfpJLHN&;_n2xzVsfNZAm85`!S|ljn_k z^Q{}GA1U8IJEEG6xxd4_9lB~FI}UsDJN~lhJB@h1Q0qti+agPiwYWVU^bUIBY(Dh& z-8ezP!pC&?A8{7LaH8jwokFpLQMS4+OQsp#@y=d6OD6B=3Z#ho+74_;f z{UrW&4u){;`TdctF(&cyHvfH;^tFdRXtDwd@9!jkgop7LWB^{uPL!L7{BZgtBQSCO zOn0&>2d~)YEMWI0Tj>}SF1i?u)Gb5^@(Jdt&tE*-fPn*YSIf!nvzJUjXbXNrX7xYSM!<)QPXoKxK26%mz^^? zaumXf68rKt)1?3x&R>^Dh9PCm%NP^}{?9{x;D8~L^&Jy0ihzjXsM>= z=6E@a5WOC9E|#mM@^7986Jf}ue*(jD%C97FehZn?VsF>77Xn3i2U{lwCBq^(hCXqx z0jPi-BAKSk$J4atOw?{9!|pyU{c_g=@RRK-CLuF*vZc##D)G_cTXsBI76G123`}_P z@|&C8&;(YUH@EIgJEe*G6| z`A9d$l>Z)OKl=1_iw13KYE^UikS2SZrZdnPH>wtob^Xy?HnT2>5rt~N%>raX(_LgM z%0CWpD|($)kD80+$otarx^||-1;nV=1ZZoVMrZ;e`DXbHuzI2QTFMertwt>V3fTYD z?rS0`kdp0IExSLlkf;S1hlG4n&a2f4nV6)DM5r$7GA0B-??oCDE{!)j)qLOH}XMIueIwk;fSC-mV^ehPw3|WejrePyu#uU~A2qlDL2_SM)Cw zD>fX}$s?lWO-fjg&Cy25n4?C0tP8LP9Ebc%knc>7kpZP~nlmGDOH5qggrniiEF_y> zHy(cN{KjkUzpX8w8M0IYkMb?OZkl7_*=xYOP}x5oZ^u%6M+m3nWs+ODpLTODXy9E` z-C~ks<^R*7DSO+5JeSngV7d{CTuB;K_%xy_Zix#U>Py*o)@dnllTz8KNnuvd*qWQu z2%_2V47yNUjDl`WeRk5ltoy4V#l%6PdkkWDYzB-CXNoKO-?rjSD&xVKCyAZT21UI*uh@eYTFNZmVT)84z=<5vg7om z19SwFN0$vnyH#7s0ZCc>RyBSxe6nQ4vOz>zB{1ADOxOVWp6^t1>Cu*aTq>5N?Pn5n z`hd!CeyJU+389^7x<^!3>FH@+{;S;pH#SfQ$eSnY(4H8-da?YnS>tWEBI|TvU=;9m zwII6<@Qh5AI_{Ygq)n8^LBU-|xaat&pWF6{*XA4odiuPEcfKYN!iDEddFkJ`5lagFfn{E3^7`?teQ=U zH@`sraGDeAxQxE%JQe+6QLkJC?ZjeJnhRf1_3sAVe1kE&ake~_$GLJu-9`hI`^fz) zyR&(L8q&x2%fC?Toq3VQzxf@1E4zOV1!1s9w%@!$&kLblsTNN4acwdJyaaKbwD1qTv)8QMbvpU9q+;^u2NfgX>G2bP_|g~ zS0_eoo6#Hw+(;54)C74yytwJ6d4>_VX+WOB09W%z!w^ScS{Z*IFS~C!qLDNa7xE@k zpO7pcQe$WL9TUl-+<5N@d8Rorb%pWt>Bc@;$4k?3(ds&G8x4M2UYWpD^Dw=?T%(9x zqG7ABV^Dr->Nf)3JzY|Bmdx;Nb$K@FTDMI_niiGaPlsYq|A@toXJ@FSln#W{R#)ZL zAd{HTuv#2XEczna5qP`UMaA7uJW+_~M( zEczzzB=)o_+n#fs;{QAZ7|FcvDk5lC;SN!-TbJ)2ib;;_Xyp^a?(kl&y}J zYg-0gM4}}&`k86Hb&SF2*bN%q#vBtsymKwYG{p^R#N1CdX!<dB9Gkq1|!(Y5Cc7;I^1sREDy@Uv0X#Ag65vxNc{uTGw%Q0>T9p%Z^aow$ce}twGR1#iqTJ}PywZ}a=eNz+#ZZj< zK3aZw2k9Tr0`4MV=rFOLYYe1Wq=0wcrYxVWJPz;s=SC0bof3-2`OBpXQ$lwP63=s zJUKdJBTczCog52|4Bb^euc=I+GpH7i?*}vj-!oAUwN#50JU!C{*N`qTct9}hOcE$G z&KG=#WC6S^y3gZ@Nt4MO1`Cz0VhfFIacN;F#>RW~iKFBfqS2o$8yH2(s5U<9qBye{ zXG9Ei=Vs9utH{;S?~+-oHc2nVEyzf)1YzrOFhjFPM>PF2h!e@-EYSzcAp6R1?J4X(O}boS2W)31@vL5|-n} zlK?XA*}3x_?7+(vw-7MtWeFdyQil4?D-I{DV@wuQJ2nkGbxl7AhY@HPUE2rcg^{Xk z&T>#2?K(AyykQD`BiUn`C~4ucb~cqW9&SQ>*GAeSs{~0_*d!aZBljzUI$kr=b>u0T z62+RTd@pHX-Jd`=;QuG~%K;syi^W{c?JdddEiSX>@tx zc$ZmjLd$`>U1#lM8PDsHWo7rXn}A0QJ{O!onxAdSpv`2g!J9d0jyw)0kfsclr~R;H zA;uq>iila$d?}##ouo2O_r$!U2PdI{^kjM;9=x+8SsvSvB=_9L0~juYw%KBGjD9B# zY{J`Iw@Gk3PuOt3xaWRCNRg-$N`)J_V^9vTh!HLqOD_(Sv@Q(W@H|yzcY*y!x@cdN zJ4%%1-D*6?WoFuju5FU1^1^VhY~o%9$w2TA$EQ4(HuH^DG6^9{GN+5amTm-`(75n= z(D)mlN%}s|)bxCOarhhZ>hp?<+!$xjh1>a4MFNJfS(3CUX&+xC|2<|9CCA%{6GC^s zlcy4cY~~j4{1`rdE;pt~%$zBc*DXn|7(yd-TtvAghX$B1*Z3`uaC@AvZEipchn-j0 z}9WunJKkMP!>y;&3?7h$XotJl6R)zgvBHIQ0Z!Ze1zA)Ab2sT}%>_b_|GyDr|c!xxLmR05NZJbtT7<7(pOS>jzU?f^c&*K@EmQ%7OwxYuMJ?xv-2f85M$RM>fA_f2@CLIm|;Y& ztu~}*nlb>$l8g&Y6IzD167>>eq~a+;@+a^ppTAIEB)p#EX2Me&#&2>CLNPSLz9;s} z{*`@p0@z97%6J=m4`Qs*YOA5wN*)}TsL3UG@x#U`lmYdOo>uR9UP(G08!O#)GY5?t zI8Ry53n@QZ`VHd@J;WwuK(vJaorCgxhgGV;P`(;2FDINt z8!cG?=l+%zhbDL0e6MWON%GL$guhxYZ;pEMv9dc_G|;BgRjFVi4Dqf!xNu1xE`I`x ztsr5e-It~uT#Mk@R57{h&`i)e9#q}uxu-2i9Iydej)B&jwedD@mTY7)gcpeMm_;Bs z33$|~Kt6XvFlY*MH_=xb6fTzGOe z2H?j2)@@YDy47VBBbYl}k)!iESCXzPb5;?FD{8Fm(Ro@fKC9#~ zQV3Zc=c9;Ahc|c_P@IE}lBsiSR5xihI(JUIQ`LsGS2qukF|5|}%XR1DIxr}8$04yJ zjT{V{f6m=W1|Z&s3zA9+@c`AN{I_NIu@$hXQ_EoHvt+;OCPzO^7sqWQ1MWrBd0H*= zVFMeEMl=zCswrO>i%&9Q!S`ivN&}u+o)fxzLA3{5`F>`zwb3+prmQFHciW)sr%kHG zGe=`ko7kh?&)amF*0$yKipJKu`Ta`L;A7%9@!;~#N;HA06{hRSGmMF`MdmGDWo7K@dHW+g!D+v11hM&}e7@O2PtMBgxN7iSwxuoy7>wM& zdsa_P#;cT}BW*~GE&%n5Ey_d@TEoDxF@_RIvR38b2t&{Jc+t2j%(b z_Xv4oIAf&neHzs_@NSjKb!4(RZ-M-0+;!lYw9QT?qYeWyd~bqq@Ij$-#LWADVRYPFUP=% z!S!S@{)&W0WsVQ1NfhIZMrf9rv^jRvV;8nu-H=gFu9#+3K!zx>WwSt4{~!5(S=?osa2{DU87N=(Mtc&cjJ# z=bAPvIVG0bEI@2rD0^36KTNQy!>g-X&BPA*JnnmNZ`8Q@Y}!t-@UxlrX$#ix+l(1< ze4OkW7S{fPYE&nbL4Yl=Q>{Zb`KNS(B&Yo{#%Y3nC)w9y~yp zpHL*5g0v^&eP$yP<$e)H&msYY_thpN74NtDxTj032>P+SfEuzk4hs@|G9KP)bC>sK zCNX*8_|j@?G9GO-8u8NG3VI)xf5PPwUHQK7{mimSwA~C{ zm(ymbh42^;MC{mb5tiopLA$WwH0B0s^d{|mgLc|LBg8WuLbZi~EJ_`h3XTzBOFnCp z$f4UEBNGz$Ofmj65+CgXYB1gqxab?%ZaF4D_`v%dBxU68iJq|ZaO~7;Nm+S~;lD_M z-dvmP>>NBbO#{A-F3BndSFw)Aj?j(oxl8q~u!%{kLz^ZH$3kGb?`i|_FKPM~2Y7@! z{^PxC3K0c2W*+-eAXXWYVUYYm&D6MSVxjeo@Pqw;yOv~RHJ(>hzH4({3@g*~N~>+x zKT|f@7&Ijj_1HpkJ2(g&KfTz6Y=nutq0&chUSC_2k@^bi_s>TMfcL^k;C1 zHwj*pKu4>s!j&1($mWVmMUU$9nTnYs^RmLg{~~-q2p{q+t-)W=XJoZ2D9`ovz#vS> zRJ8)M1B?JB;vgyG(xFDE>z<^n>MXD=0Y?qoq+{6bd1!OD7p-zoi-SfgbZ$qo0j;PN2$~&DyHmgnvj>+qRhea*2m+Qth4M`B2Z1f!#8znr#DXWhd!@7Z?jGm&+oMS8KjnMw4Ir1pU z26z~C`iwT$-b4sEZw_GTP|xbm(1F$7uel9_w)nMrrv=Fyqh&=!zEXH+WXdHEAMt_( z@^YZ0&%gOy5qy+Z0t1f>Uks3dSc;d2m+Qz#2o3UooDqnZBV`D;?zfKF9R*N^-$&pD zhP`R%7_w-st;*+99~t~AOz)K;%hA&bHV2I|c};ZH8>`4wLEYsf zL2@V}aaC9Fkd&Qi9f22+u7iya-btUhUHD`Qj|?x9>6sCpv^gFNJ99;N{PSUXiHN0wi^Sbl9`EHol5Hu{f^>~FORGM#&u z!SD~>m$)#r4!Vs%*idH9`LvW6%;26O_v$X-!}NPfwZ8ko_hD6 z4tqeD#J9zZ3^$T&_Q!Dh0mZL?cBU!E7y1kepNdagR9Ry!fxGzK7^ zSkq^1^&gQ!(HTy(a4E|;@Q%QF=9v&4=RL#$5M(5Wv&!S%dyHplT1Wd4R97*n4u~b+ zE+LU5RtTC%mgM)2XRZVihoHg{m38RrPStnjs5IitSzuF^u4THT!xdR~fhOzJcQdy{ ztocvo$up)K^CwMk38oWpx#Jq=`Qa&IK;co#5?*0ZREzZ;G+!7t(@V#sH9o~x`ILcR zkDlRkrm+C|STr8g5GbZ?D`%3Y)?0(Wc*dYSM`kx6UA%gPb!-oSmyp`gg#Pg;(r#KN zZzyBsaa~D39Za4@8pqFa?4)Zz6EW%$Wo*K6%2?8>DUR?^k1WSPkA|+e&sjYdc+_@} z>9bo|j3bE%rGgS!_h`a*X+7;FW89?Y=#TA&*BqXkDFKp@GFi(O=hl{=!vNEj(?iTz z8BZ_8eV@>43OP29jR;xK7&SE8`pOuXHk1HEB8gQgxU#XZM^;kyGU_oUkw&dOFD=P9 zx7vWU*(5v`jOD{h!T=%3v$jXKE9^utChJgI6%>XRv>A5Ue{c`!tMW7Fzf?AnXT`9bw-p1MTRMI5eOcH`FtCWW13Z9$2{!l=!n->PtQr-J&i*BFSa zafRR6xthj;U(?uX(YKT{u?{J>5aF533t(yF^MNH%MP3{+`DJXfoLd_{;`OMy}uZHQQ|(IBm9>~l1B;<3ja7fv%qVF zF!V?YPj$P1c?+*{oDoPCuFs(iqmI$Id-R>>u-~sHM7?6kunEH3#u%fi=WPnRT6x~+ zQvq^3hrl+Gq*EvXNkDYjl&pmTUt}G55}=6CMaL;LH;mIVt5^l_2>q%T;TM5tke^qM z$ZLK`ICrn0jiNZ8$@{#Wfc{?DJYNl(`BMvJqA>vHLYI6%+cy=4x?OiZ!UlPS3GpuL z^x0^H2PI<2*_4LkMX|hnjYa>P6Ul^kJex`04i=TQYVuWmx|g;zrJ6pKN$`NGUEN(| z+a?t}kDeAiL5%Pq@dD2)$r#eyyXV;mjYAiZ9E?siM!s)47th%yOOVf)>JnRIISSPt zOntJ={!Bol$o-bOJjrI7Im?$XI!IUG@3*W-4j1Pn^D-^r%RPMuE*%-PG^9jy?=6|8 z>-_f{!@PxM|ExlSw5-C|X<+h1LYYwgP0dXsg2DZrtjXONvSn5ROSzcpqz-yC~kddY!atPk?s(eQ6&cc)8o#DiURWOW3yt%;q*6_5YH}mq&al^su+HWI`7xQwhEsZK!rOy4Auwk7NvPQu{Z8aS;+2Rq zcDPqr%1VWyCV6AVA{vs-oFjluX+qkhh2fT0B9BeRrfr|HBB}aDs~7w(2k+Og-`Wg}+~P>d_vEnf zbs8YgCvn%3stU04HF?fx2Fa79HtOy+9K5Xq=9G6@T$KjGWf)pvei9r;jMJ6bQvZWc z*N^dn(fD_g{++27Z&0sBJ$ zZNxMleSNg642QBx}P*P2mt(mb`IPMLqy0 zS~nK{22+%C8t5~UYzCEqwHI2#Wcu3BTCOcdVU}tyaZvNj9&Og3b|>CSDFDB$xkwOP0}m1a!gHu@Tf z73UBFJ$Q|V<9Is%L0d!gq?YwRaS5s)J<~@V?Qe;0P*Ie`tbYLC+q9Z_hcvU3N4E?3f@K&*VeCR z-P9N-Y)W;kq;uRD;hA$S@b7f}ViT<1)3&8x`0HRXuvxTerZ6IaNNp_lwO9;rF-< z@FZYJ>U7%}-!WO9Nq}eh0QWIiE$*8=M@iMJpBHJ@(Mrjck3F`W8r}-ttrsd|i z1*k*s&^;e&KBv^LtoQZEb{Z6bxDW7r;jN)%$fg*Y0GZjo_G%{BYWtb^g-a`P5q;ZI zmml$2n<_vyJaTx2Dc;$`&?x_BAs>gwPa>X1W#nCjL=STja}smY_NhuppUv8zRv{HX z5h)jpHL8B_bAc!C)_s0>mYlC{hSvMU--j*AlGz%Ql%%u1pf&(y+5-1@5&BHHc&&jk#R+@x_D$72wi;rtzFN64H>+ z34b2mE*pCUtdq`dQ)&;xtC{kq%+ofJ8Xa9}t}@CO%#vNqP0}HoDqAsm3OhHl5W}24 zEau6ngo?zAJ*H27kQBOWaGxx_Hq>`C5nO4LmIczw1_XM-qlB&?>eDcY@FrxeXb=lD znQTIVdDgFuLqh_kCU=m{NrMQS(co=V#ZZR+lNe2E2lH4`@^1CH>gkW7G=MdROoh%7vo@~lWuNOc3m z)bT(Na%PdHk;H2d_N@*&~Ocy|{h-%vH_cyO556?Oa6CQ#{vZvb4SPEo80QRO2->7FFd`A;jK)2C zj@NRP`1VB{N0?Gr*AvQN<4ZV=BJygkyDen~BdEiTA}J?a-_++z<>s|UEI6q064ZSS zKqU$jg~n84HVb0Jj61?J0@QnZwN87~&vH-X@Y!UE`^Jw}riTpI+YHG@zNfcH48vK} z^?{8?T!t|v@J!k2jLoD3KHfK7D7~R(kVCy&^}HU^c{$_g6kU%-7NSAQ1>fh9{U7(tk{Ad^GBV2FV^xcyZevcr2n-I0A#)wo-_X#fnb4eMpvvIfwxNMaq&F?R#X#sTXBYD(EK{FMeBrlgb7V^4t*K*mvQ zyw&dI;gIgJNeY0ICkgK`28Ahe7V3d1d}|};RHWWy7=*Q>_XVEgmH~NYBth=PHWVJs zu#JJI=Ao-F6i@1xzh?s)Q)#6s#SbE3YE%8tCl|{v;Rw?{zDWKX!TlkFEL2wlZ=KD< zdvAYhvu}rH$vGrW95t0e+IHmsB*5u1>Fd!<)iG?UFktMp@X~Sx&lY|k)m~^t7^}AY z$WsRzUYcnOvc1n3l;@J}k1sD#oSxJaJVc%wA5N_)w1=4QGT)8+t<=r@+#Wfd zY);cX^Rws^xo1>hN~%py-C&#K8=GebKQE34Tm;ZBB)pAhlI9zpiO-8og=sv{sH~B( zOAY4+Mkv|$yqk@ZeNE88SZk>DOY7DshZP`TT6zEvx?va1k`vX#e1H^Vny-ab$_vdd zY(+)BvZ^X~;LTJIc9qTdLRo!xFp8 zO`@Ju!YbTI%;}$bD9x?Z_tOq#MnX$G&;xJ`J13A%cOagk~(n97&I zSsY82JxB#xZ)_(?rH~XvqQVBhAU!+SW{*j>wUt5CD_hbWm(V)&4W2h4EPi5ahePi0 zQ31!+H)ZU7Pz)P-!h?zH7B+0dS=kp)7uR+rU^5tHP}V)32`3EpBzny#7W9b{ay zfX8vf_)M9JN*HZP<;OzfFHN22K$c}qntqt*K7JzR1F0@frYa+0hz42kry~b6m57nM z5yZ1jN~p=&e}Lmjr*uH|jwt7JV3sU``V%!8=?65I>6{Q53sV+iAmuVqkuQBmLlVUZ zY?VG1yT_!yV34k^%6H?wM_tvVYo!W7GqqC#@RI3-Xff{CZR?I{lYCz0I+|95 zWt-7jvN_L%7K<<9$a$U5RLX}IS0$w#!3HIo6r?t1Z>szC_>n!bGV5{i?%6Ep<0fV7yR-eBRnKnz1!t)M4mmLCTQ8Rr9Z&F&)KXVG4wDA0ffx~=bNIDxf8Fd(kfgT-N zd8Ymcl^Zy(ZUM*R86&8T6x7$KZ~{GqpG9~Jd1rJSK8y%MMtCFpoAJaw@ikNtV@~a9 z!+_$MNBe*glO;J}bw$2?>x#qQKb}Pz6^b-K^Oa2q=V09JopQ*$>ho!{oX?8e8WE>% zY&dj={mE0=@Xiw$g15eQcoyPU-pNAbC8&SE;4;Sp*NS8j1gq7q`j@)CP@p=RxolP2 zb68&KX<{)AfT79a96dhC8@hV70U)KC!gA;gHrY)Np-8+mkYVxA;8yp;K3UxLS`1F{ zZnWO!xCOHrP~cl9){24H2|Gfph|Tfl&616#y1p0` z3uiUtFSO^S<+veepzI`&Ep;=$w+YgJLr0TEkw$pYRFjr@(IjZRKU9v;M*hUtZL8aC zWOocUG~yb-lew)fFGM%#{GZ?F&DUDlCp++x@owSl;5=w7BEYOm-XI}@{zQ#iHa98d z&kmv?NZlt7?NY`#Eh{G!-l0yGiaxJRaFFDHI(_!(RbOl7c@6f;78{ZMaH0+l4AY!~ zgQx|Z=FY<+PyPgMpE4}I$Iq|SdpP^Ei-Y3el ziRD6MQ=X8&t3~K%HdRo<20~saQwGL{^xtght2CIb$=#;@&^G(D6;lZN?kNHCTk&8< zMV8|lndah)XY0hQF+zPkyxhH47!HA+VN_!=3X^VKmJoiPu&JzJ>)|dFw&m$AoQi}s zt_3|zo%b(jO(djrCAfJ*qDd&AR@a;oYg6Zggs$5qHlF3&#?C5FSNXWfGepyst>d0x3`GqL#wPrZ zq#Y;5EFM$%ZgBoEKJe12XML9?S$j%mXFwj^HYoeya5hz_uIDg^!n47T-URn?Zuvs+2 zg2ud0EWFV*fsIyR{9pjag*Brs$h0M!)U-k5?-`Q`as$uRv=)*CW0`TCH#{8|G(5fr zQ*wGsfK{SSSYE~^p~kljQkR45Ixx|pugAo9g?b5TXz;0p?UNE&*bt01C5!ksws2A} zeYVM5j+Tu>o6Yo{M(sv(h}MJ00|$=<3@_TPe&|#!ARo#5dP#RDTbY*EKXHAy({8@5#3H>_+ zy}h!uuSd4i_;>X0Bnj!Ncfb)=8~KCxnN6KZl$y#p;;i^QKa5 zjcOe&2wTke{W+;oW~c(}&A@?R_yY%)Jtd=o@~J(6WG z#)9e-BqvPPWRI!NF?_rCCDddzRpzi!oJTJ!DKi!?)~6FH&Cq_b3R1Z;&U+&5O{QOuvrvr~%QQ1Xt zI7U9 zl!5$IR>O*gCTYIOSVqbt+)TI*W0;0eGYJW4xVI#Dz~O`6OhpD0e=P|Zup~7^uaRp* z<4so+*MLNhB=6O_>N^Dn@ciq%J;69niV&Fk5r_$s`AZ|wV<^LB-6nd{M5ua+Ouc_g zGJLlIBztrv-S$qck7b5H8(hygqb`o;NdyLag*AfEHI1mHKi?fTKj|cC?wpXfCR)U; zZD$xWdTDXtTs3KZ_MG{LW@%kND!+O6Y}q%jN0w#m586qDz>Z6E4J;yP6o&hZ_Y#kd zT1bzI`x1kR9edP6!GN*xfjb1bV=l4->QiG#8TlT`QgaevYwAI(cOB7L8MrCSG z7F;}2HX_%RHES2I>!Mz{C?)3)<%WNV>V=YY;d@0=Z$R0&@vL_aHj=AeLK?=e&=W1k zmHNw?k?c5|>!q2r1Z@mS8eYz%W59)DVF=#?yj&Q6&1QrL{K~ld>rcp8!pjE}XE3b! zS#;GP5`A>{cmMkI-VfeolV{2w1DV8bvj9QRy{~wfzlLFggcFT~ifD*60X&~z zUCCN~BoQ{!Kzt^LQ+fYCd+#2sNmbwd{;C-EOz*viVV*t1W%dlro*T1ghO1&6M1{ez z!oXng4OXET;|S)cF(oK(6<$(Nqa_^h7R`xx1Iw5~!3qYGkaM&c6i!sM!b!oK5b$7* zNlYrHUVM$w{FAdj{j4?g?&-DGulwoSv*-C%t=cntKTr4S?$vAk*6;HDl4fGogBz_k zwziNr26hq8DN!vv`*oeC5umfHF#uz99HU^ztX!8@6mg9(b3Xh$3P`ZeC%EGxzm#3* z>Y>7k7Me<8>kW9D=6LK#%+BNqa=h?L;%P8O$6v+hNjN^ABdl!rG8c#bg-0r`cfW#qq0UMKe|*7v+B2p*A`(P=$*K?3m-24Ijr z0uL7^NJ&MNwt&Z&P_nP@ zsFRrW`wd<9GlapLB8if;rYd>~_gxd2s0mSPb$vr{Nu<8>P`se!<^#M8F@xZpr_2A; zkV%I~ibFhnkXZUc5QGrQ6ro@&;aP%f4X(nM4&=L*B(rcqz*voq8%6bSWl)jo3qfN9 zBQ56nY7+8z(s-`t<_LfZQ~RXKT*D}xi4dN0Ph$m09aatwcn@OF6%Srp{jmxQ-^Fui z=Amzg#-ziF6>qxCVH_HMA|h>S?@Fmzd2X3q0>Cp?({ZE$v-v z5D8)NRGnyLNautJ*dl!m6`M~=gqm;p+uytB)s?xs_NDG0AJl*`R ziW-Rr^RZ1lZA9(kMxpUwhN#SXflv_z?GJXO+L7;U;@msTA|9Ji?P55x#{nvs7%8o# z4pG6%2YJ0U z7z-GaJBz+e9f*2OL)?c4=j6$vA`H?7eBt9y@09=0ESwnHG08Q&SEER}Ib)ub2%1ep zh&~4jSTu`djnac&5Xxr5mN%BU{|d^T4Y3M9T9!Cg11bv%#f6SZ=%+^F^d!@cg0@<= z$g73I&&^wk=N+pTT*b@QTMz<_shrTR_NO~F3hlFOnz%Toxy#?`s%l2sFLu3`_f(%9 zZp-&BELBXaV~%MZmd*U}B0jylhvA;DG<9w>n7^VK^m}V4_s6ppJ5Sl1H9TzBoGkyC zH!KM&Yb&8OoL7WWPn(w4WxuB>-e9q8iYyfmTgY(50H%~op%07+c(pAkbl7kRWB;!6-s6|>t>^d$!vAidf5Wi2`m@(ng9xnsOt4GL> z&>{vmm8$td0H{@z9>66FHr}d3k#H!g9LSuFdM#~%f)Js|9FVB-^HzMjVdYd>wDNpy7jq*>@kQ1ghFySysV6CzM(a4M2 zHN5$aP4wp~iDEGnM?BSrH*XAk%V>B%Vl*CkIB&~Avs0lFrrKH}1fbDhEt7Ds3eok36giGSL%GAd8cHwVKISW9^9D{;Dj2hQxSf10#<>%_$LKzL z_^ypKUy@^m=d&m(L&$nDQ`ltYs)MmmH%tUJbb18PW5m)BxUL%Mgc<|yqFEpA=Xvrx zxNn5BEgAoBWAD3^`7af_IHPw8c>mL^wP)CZJde9fR=_ zGQRNo?wpltN9}6}TFM=m2SUezyAm&N)>DqLhF5KIbe->%0fF#3i;IT7NQ0P>LQd4C z40$b8zCOdjr$wtRW~opR(Kb?UL0@GyIc8a}sbxGUOxv-c&CYat89$J0hn~m5IeBt~ zA`lH!UQ*H^WCVV~JS>C9BV0d?f7d|Ro`QFU`Yp38nuqeH1J z!mwV~!ve=Eav| z3JG@`sxX>VslMjr<$>LBX@GPe0}U~QCXWB`dF={fj4h0Ta-mt2{yv|B$p2q+N;>F& zId+UQSIhrFh&Ou-9xjq~+o$Cf?mVFi$6G1co^D@UoMHnEE?>WV5@YOUUmlUk-v8?gx?Zaj*N z0UF`z8jBl(C_%Z?r)t%ap{+8`irzcm>44h}qm7>-;$T9;F-bu@CfWOoadP1r`8EbB z-=$G7VK)SwEOHksxH;+X>J{?*g0GLNa?=PSXjl}_EJxT}D||eBOu`s(J`X5)WUt?_ zln~e%Z^&oGNP*$^gAG3Pf_QXsj|rGD`1PU%*-72??7pZfU$0y*k9XbC7%X?oU-{HVF5fSu7Y(7Ur3sck(K^idI)T z$gzkZ?LiR^&V@PMXy<3DQ8A`=&&nNyglO=+t*>z*p@Nk2acn|aTEznfO&W$>Cy%j6 zlf-OyTEPimLn6Quaf97i(JYg*ltR0{{Sk07*naRI2FwRt1&WI25wRHAX05=C(ZO6_A*D z<$6~|4c!sNCU8XJFro2pTYxe)Ap(#IU1w%f*Pp0$^tK2tY6UGNK0NcV<6oW(gwz`uG8622PlT(x$|z}eP%MVjcK_4n9#Fh5u*`ayf%`^8^B=ea7?{p zOOkq-#>NBT!De|j_v|QzgALR29G@f@&#|Pocp)$#%{%><7)RuoQ5owa3nwQ`6PB)F zlU*9~({ffwJ`JO$MLpK!pe7o1LVrB)+Y|W>9~n4?V3}|)1PfG$TQ=0Uvu#cSjNV)9hzo;F=hch`sGkC#2~R_F;{HYeY$aQKvrwkQV- z-lE*ZvT#F0A%Kt}q31fTyDn(thcvQe3o~*dMFJQtcsyxvu%Rm(%R|h=4TqFYTN11TZq{dm`?{oq|jt)ytd$NT8= z{v5U_(nM%Qg(+X}HO~Vt{CasIMg1ZYt*@FPaKsmnw#!slPY&bN$gN)Y@u{cI% zOr5DIFa~^U+&F0Ujy2X#$u?u~{f|G)S>edb*%IlIEpJpzLW7vfVCZ4W9v2mhkZ}Yb znp13Gfz@~-nthRbc@};*Pas3=RtqD$YVht%TzQIO3mkU zW*g_wYGM+`j$Iob9{X|C*uyId^^4~G$2!=gnDO;!UU2}fJY%@9DQk2=W&sLqChu*D zFjk{CYiWWuXTKNnqNGO6sq#ZAg`4hwpMjyQQPHTOg1Z)w`>Bx;M6Wk?ERB2|8xM1j zcC>MAMZk%HV}O(fUond@2;ay&%shd8LWMAfBa9?{OW*1}2>ZsoVk8&Ot(2FJdam8G z@-~z{?#nFvW`X?A7oI5Jh5xVY@wB@m27Z~(6aF6cTAKhy9s{3{CRKC%v>~Th^>0Gu zaqPOOY>xHeprT?cmUCxb;P-xdr~HTS`wtGi!C%+j7ee;87swQ`V3#2IRC)BWI@AXS z@#;qFN$5Ivh3ns7TW4Mka!)BYnkk~YY7H6*`y zu}(iaEH?NhV?gIORu~Y3sk`7$?IU8<(snca#jg7aNGFKN0n!xO$n)sk#*UN9tgdRW z8oMmWkM6 zTjU90_MNzP<~X@{^^_dor}pz2L2UGHWf;!#;wr@=d2fWXNs~K=cTGwdP77un8N>ZM z7`imbk+qt^7TZH%H`ln2?!H4M5z4#5n0a1A(yd{n)qZS%AgiU>4Fk`nHA4_X&)kQH zvIR3t)_95?uP1$qh33fJf`X-qK0-pQsiNf`N{#YbMIbFg`HmIkeI<;0p0lGR!m~-y zN=@Bk++1Yao|=x?DndpC%|;-~Mxz=5qev^2L@HV5yGM>IP(yXZLTB$m?O?pV zM^UJDT?=$2r$6Mf)jvu#1+S2m>^@9)jS>-z=w#To!!7ukb;UzjmB~< z!1EpQ0+P3Xm{m8=1nP2jGyP*|<>ZQ&8)C8*Ve`oi!e4bWi!?`|@=aZ*Lvtj~G zY!1@-AfFq{Jcp9hmKk}8>L3x#U7{|1&Jksu)JQN=IMQ`r^T6wTgU4BQxJ$05 zm~3co-6a3oq?{AR9~!|$=Vl@dsR=Ns;qEJ1xLJ4to{lsWO~oq3A?sJk-i9vE3ycSh zrTjvT;lhhuCn>dARNfg{RFL%ArVGahJvtadv3_eRcgyC^;qmivNogySsXjLM-CHS5 zpHOru`YxVE@*ZiUy7d{eA>)nl9@!uyuMF_6xdw^$DE-uceA-w)NaMg;8T*{o6>f}N zdiSh+EW}5~`?FYs9F?*!cNOJkWBkDfSf)~bRKyHYTAw{fO@I(Fo4jn=h*J{5!O+Ax z;k6~2MS%)*K0Z#!5HRq}JNZaw-4rt1GIb0HUF5DxPpY5K^^Mo16u6*6wP(??wB zgGukSy7hfoREkt9F#cHlw8x=O+;0jZ9G@Ty1UYyb^#hX8eHG1ap{Gh%+aHYgcSiDL z7!ydg#g4^J4bO9Sw~bM7>6|=f7Q)6UD+oo#uElaye%5!4H7jMGCg-y%@p>5?P!tW( z&YF>L6czGh8#E^~4O8m)^i>%Tc)^-HcMjz8-NHOZ?junVlK9V?l|OIW+oq{Xe*YHv zY17Z9M@TopzHlnMb+O1**C4&bc|NyJ%S|#KX-^x-QH+g6s)4)iXcnti$OT2~ZoVlB zc^X545r`Mg(ZS$_dfln=L&A|wo)`~g%&NyoQqnKr6>+4(wG3!HEf`{X1DQU(euMmT zRk;B>S9vpVg10+Pu`nsg=s>E4@z^ynC8!IT)rRQPKXi5d!9U#wI`1d1P39 zj%Ve5JbjM)?C`lqG*4ArjuX@mIObww>7Jnxe|^_|Z3CJj9q3dxB>kmR8Dk`Hf_`%6 z(qa(i337?en)2$}PiX9*9nZpWwatSn5j~yDmZDXWoNRfxem_ECL(oEaXa#hAQ2Bav23@^N|P30=(kV-ZB z`cb3F|yN-+oXtLWFO!F2%%KG&KoaO*8Ltddu- zTZ$Q3x$sYd@I1L%&U-`!Uls3B2vAOF1lEg;G>QTsyk)YWy7#36`K~QyNe2y~zL<6= z*KI3UCkOze>64K$M4CaJTDv@7o(*0_Jdlg^<(qE6s~ox(tNj@(Mo2KcipH;1M`=xe z6`g6)DDt^=BycBqlyaRsyU%qP$YAUhpV#8VoT@(BlT6M$oHvMq}h#<&sa^#(xo9P z3vwwjm}y7~q72f&dBz6r`Jg~T*c&ezbWl)}NWt@gpAK(?Z$UCf;GZ_tSNc>$NKEdZ ztpM(Az8fB8xYl@Qxz7|8VnlB(nYZjf`c8~b@A^&0jz@Gk0*=jC!3kk#Dzml6WZwS0 z?ef$7e9k-FMCf{KFh4D4U%5`6DhdUYAC#phS)7%(hA0WFWK=DJ=h|>wR$x6wC6dYv z4RuY^Yw=aXwwUz3EPzo>@koASz;YFCbAZbDS^1#!*!bJDN?vft6MlpEyGpe#6fpY(v>x{iX{YXJ;BqEt z2}D>z#U`$uzq*{Qu_ckK&F14s=o-&`XC%;Cm>twFIqA0D4dvu;Ok)_`jYT-ZWfFBsa4uKTJ5-slj8jwJ|hoc}Nc-c#tP%-H@S z^UfyoPIlR!8lXxF&+>wqIqY2;dZ5m!Bzgo*1n1#l5nolRvy`im=R_mSybQ+X1ftLX z-@A-iIwZu)MMC3Zxe#T@uu~sHLvN9l5hMAsC|{68Pob!gf(4L}7@j*kb@2CPa;cW! zz(5{}^{;B>a0ok$@kod;?|tMsmF3kN5u9g(ZOaN;CVD*B(&MmL1`BM8V%I~FOIID_ zJs~8W&AGzMf(C8xLnj_U4-zYUqmdgZ_PyP>QeLYWdr-D?e0jYA`IWKzl{d5}m;k^4 z5+iRWL=4AY*L`gSEmR-cnm5Hy$ie2D!P+CAWwG4lN?U76-$jS;+cf@@8dD0 z!SUC3M*Nc4jt4pRd=G{>`g#MZ6hoD1u65Kj@J0Z*Vq9bCN$20azC6iHENL#h{wJrw5E)X$^N$tE+es&v=x zG}rdA4O6nCq4RbRv(QD0EokSTAYUo^K9>5W19>9!UH)DrjDt6;XyL-6m(|bJNElIg zQMZ>0ni}82N_(4u91hy>!A^bUdU8LJ7~P3PHrpzU6(g_>6|=F!+3!>)Swl5>)jn!5H6=T<`Mti@ zg@Ow2WK3?0ws*~Vdc4XOfmo!`VA>56Jho3T<(a=~C$OLh)enGe6 zm<6s{FHd2_Rax`*%*eeyZyfV+RZ1TX%vgN&^G=Y%T#NbnkmX=qo}NDk47=Fm%LImG zT+#QlRCtky`u!zRb9|Z zM2@k1QfV3(iv+LdC1$L-jUmMc>l1l?tG#akyc5jncVa{k_!w%4}ClKJtZLFH3a>B`0d8lH-W9#8~y z?rOP^YOoIF`%1oB*WP0Qy?9IiGZ>8;aplO{ZE-toF9cUlLF z3p`+XB4{dpWx0D8a=@td&3SCDLa9bE)J+6EK^QP+gw~da&nuUEPo-^a9=QDs`T4pj zIZ)?y@P59;7^_93-<)r`^Yo_S2A@tZ?$|SD{l=SlXn(3P0Qp=~7f^Q*NL$cU5+gC^ zK2I)}&zs~LyDrHrDi^PnZ^zDY>0CVd9|QM`3Iov5QxKkKM@2O^xfL`#a^$}?5cW)B z9xf`U7iQ$bn$H`AwwkytuPJE&g#K9|okyK#UxGqtz+}iXcHy2;3xHtak5wpJBe}A7 zs7Fvs!7WEH{$lx_j0L=TBc7@jDY)ehsBGZig-eyTtE9xLrVtYEE3{gg1Md33!jNTcMzMUOj zGjiTgC!qQA-(skP1}5Jx^70szv<3{90Sjnsz_Y^=c~<<|44gmlC5U6>RLl2fo7=aUFW?k=|Nlq)D(PqfXluL?-+t+t!QF_pWtYib>yq8@A-|E+1x#~Magjp z6;W(%8uv|@#vpY0#J2XtTYE)l*X2Lr>opy)(JXM%Tb=D6Yze-XAq*-8KhoU)GtdplSYv$z&kDYb?s5tb<2YxuRVP?%FYB2g;7{VBn*K3N< z>Y&Ji#x0u?R7d}NfgQ7Q-Mc-?3H*LXL1ier!Mu@`ZK*s}^}MaJQgBDGYgWDxGG4IW zd64<l@4S1nJoCSAlb`XE|MUBz&mUA&v_=-}0Tsd* ztdaX@yJzKNGJ05XgC;C%7S1KIFjU#%Y0Dc%aN~srEDLqWt75qhN20&r`yu0qK2s^H zJ)YHaz6U~TJT^3&(C$j+7YHfG?jd8u=_HQ|4B$M6Q&B4l0dqa z`)&4t5|g8l>Qj;bBkz8qP@G*$`4zzukK5)6PMj~c(@ zWp`YMRpGpLp+%d;ufm}*wzu1seJNGBjTj)Fl_p(AOA620R=jSa8yAdgc?*ss)X2w3Lhyn)GV(&S)LZp z;GH)+5WIBsn?;s}2aRy@{EV)8DhBQuA!)oB7(2Av&yb%{!MSs_TwcdHvM{^H??&%o zPRxE|B9}wH$-N?X^NKln!Wz!1#elH9`R+dyor|2(d=)A8BqXa>WC)Fdl!$w^+=aIq z53(=3iNTJ-4SB3g)I2NlJlQ?wH#cVYx}9+oy3cS^W^-^ZSsN#Qx1PK#R7O3=uIqla z3DQww?>*IwnNZdM4%>o^Zw zrV|5jh{O!mzl#mRqPRPTOzeB)Ggax-w3*;z!qBuAuayVgcQiS6yl`1=z9O*eA=E}| zDvX4Y<-AHwN|6x~zdbQaoy*%H3>r!$ zfS!owSmj=I4LAm4_avLl&e)Arvzkrnk7Pna;=_Q=6;-5oc=%anvv(|FpJL<2M%Sz# z2P20{4$jGwq2n;8RM@~|xq8QL+*4>~=D2(D(iNmO)dEs=vhzk=#~n-H^$P81(n`_Q zM}<-n7UkHBfQL{jP9`x8Sd5p&XDMw|G%Mu2P^{t3LBR<5XwDl{hCF&)fQm^`Xn^}? zuKnxobFmOQVI|8;4f35d_wj6- zL&^qF^Ujm=*7dbbuson~G%#0c_%hjy`qDrSKg*drj;S1VAe-7$`>{#;nHYdWq&W?~ z{5KX+IeHN8De@HM8=P<}Ry@1dm%*kP+2>m%tueC1HLlU;#UmxdTt+Bd{v=O#fw#;U zgWt)@+bsv+1tZLikxa`Ki=@B+j@`d9DlQ?t;w#q4leS_`wUMy8%4Micf`rtuDmIN_ zk@Y5XFO?p^Bj-GzsL=LppzJO=EdX*v(&P%+MJKdiJXeNFFwZSLJ+ zclx*^z)qb5O76o^L3EE)r<=FRGi{u&*U09-Id2Wd7aVaO?k`mc>(G$aq;fdH431Y} zxbQ+T@zY$t^5ICdbYqJ!#vPv}kHhETX;T-r19%HNGzN(^#BPY4WY4VJ7UH+c++&sD zZaJsYB<3`S&pdzfKd(ATetm^yIo7A_E}y=|Dz3@3 z%960dAsNbdvzxD2E}x1>di$(gM>+-}Wv-i_5xN--5<)4wdFDkA*DSc{7zdAI@2)_ak|pX;rSE}^5f0C zG7{rP2sQ`&-YbXe$I}e7KcN{$CV&E)(M@eEFq*;h}=0a|uJ zEdrD!vB|e;4qzV8fgry|3vex^ULRk66b35~pNiL*>;8YX!*8|I^EvwkX;_-`u`(@BGna^ADYGdKNz+VMid!52IKL+AOc!Zk)I zMMBwh0=!H^bu3DtMc$d`Q2?KY*ZbrJB)`WD#F z4D%arcAq7GEjD%zotk=4$1WG@7(;>w86_H8TV9}wsHF<{SO!id3^3n4j4*ao#>N@B z(Wc2bvHrEzcwA0QEskIu6_D#Cy&CE_&a%CO2N4nhZHX5JT6Y zm&`<)r##3v{$5t$0}2}!y3ld285ZtTF}wz5hyJyj%zbndH3lJ`#sZGDAf3r4&yb&K z)ek1<2F9Sj!EAQ^HXFUUd5b(_-hpp!u%6z&fvQ@n&&`J9h=4*_9>8N6=KcO+OljoW zVewXNA#iL1DvXQHRo6Ls13Z5idA^(b&2v>@&Xi2pw!+V`5a3QM z#uI&G!H1s@kL4Mwj@=`R@vMBN>iH*V9$ORnZ#Q`)GRHxCI;#XiO2?>9m16~(VlNy7^Q+{Qb)6H( z&wZX0vAWLD8|;{o_xdi@wL6A4xTxwiVb^@=oIK{VeBmQwGWF1aB-~uWZZx~&KI)~l z{Bo&U2ro_EFl!4W>}x!LKJQQ!*lR!R3Dete<%d4(Yb6lh+!=DYy~Id$DKNZbcP9=vBv ze0TyekTHOFjzD~>y=SOrlsT@pTDMIiW-9p4wR6xoEyjd9scJ^Ogmj3NE7J}4v4A8* zjPEul6C-~4L0d_E#{m1sG!te1yHv*(838;HPepJH1IpP+ z?#v#KaH6FZX?2IEN!Z7G5#k-fc*8)1UoNGO2^e#Dif-8|&*aa6A+fum1yYm4<|*h% zC~Uy1ivdDo6OgBFi6KO9K*LLn)^nL_(ioLAr(a4q)@SIK$T1o4IdU4Ckf(D%yV+-W z=Q#(k?H|a)UH7#m zkS-A|=voed+&cE$QY`MiE zl+d%_u5v8hlvFJA~J7`uPXdjJhKo1vNJ3*hI_(Th5Enf<(pN!DP2vkp{L>RTT5!b5|WKoLZUH!;tF|T**+g5NJ6q# zf!xJ}ie}z_TiHCeh5|(&nn;YTs9m-|aRs#wQWj9}K*a-^E;Bd(PS<^F3TE^FkywP1 zd~d07d_y;r*EL?Y1|X&C-;6Al7+*VbiUAtWQ5jVRa+yei4;>G#z>4`!RF1ukFd2(F zjj**Z1f2)rsjP88V4!=GG_UGJ~XJ2VuOCg*GD{Q$g1VfNq$_&h|=e~%TAa~U(tugg_0Gq2&9XDq-ON4_he zG!5Z!AF*yqF4IV%A=}k<-!QN*SR>!S>yHOg!LL~fG_z&OAgMBWhjLl`W+&$0l=j9LDkS*Qmgtz_3Sva^9vWxTph2t+5!h2(yq=6avjq73u=bCV_Wk&&<*3 z8X7>T&L$#X)y)6-ebs7XW01e#iUO7a-pOAYrix7D#5~YQ-ena`@CNzDJ-mr!i+Fkh zWJrS~@E8bX(-VAPuE#OLn#7#%(Z@{i zpaHw#kb%R3OhR>;W5&Rw5u*m-@0q;HMG6gK*G{Nx#R@q;e?AQr3Av0S#Yy8B$i5H@ zp0uqq@{&65PZ3JwN}@@6E8Z%nRzjgnk#4W^`r^SvD$S7^w`Kg{!TeCcBto;fg{3w# z8p;~2w3H8y50VW9_m-b2$@}P9*J+Ly6P>0~HKKHgDM||KyA6B28}F~O9A^{KUasu8Oyf*m-9d8bNkrZKc)i+HM%{{{+ zt?SQFQ2CJC)C6&E8#r@DzLb~bWG*ms=Tlp2D7b=3=pIjjmCZpn;Kz92(2>z=PxX;FK8n zV)^-n7}bvw#|?t;fyM)v&D-E!g_7lM!Aao`PgBUj_Z1V;fLu&k$@^RxWpX9)RE8Ef z8`^E!U{yg&AylTwB*r<&GuEawOmGB2wT6c~mV3l288hne`k9YzhN34n-e?$=U0h2Y zz%erIMGjfxNlci-Tbq;(03z3RSpg zp>y*0ocAZV5ZMI$!h#cx$1zK$+!vJ0l~ms`EpV>xUAnTCkKEr`o zN8hHVWHImgENU3Wb)0X4V;BpGy!-}_(ewv9j-nhWB!b4XRdFFo;QPz+!r>|7xaX}- z2Y2aR_yI)@jW4u13U<%PmwgM1uP9@FM$U997fzmz7oX}_r(IhF$hCb`HLj5}SNAoJ ziZZ#97H&z5L;ArxaF?w>wy6ib+6;7s5?aum zdsld#gcV=7M*fV7*Y$}!KlhQK-Z9N-V0Y2~J_9dmOmmiI*NV-DP7OyR;Ah;P$Zyou zKPq8~0u{~iY5wz=Qv6KNBar_W+d<~4SH}SuVh*a%PUVRq(G zJkeA8+&kK9PLaRode^R!O`-21QRSn^ z;rD%$up&I@JXDq&tSwK^fV|Hy=Dc(m-cX5bnU>4)UnrX^3GsHwV5BH%F-KTl7ee`| zK3+j4Y&u#`3FJXlJKnSJ!7^}6^7Cs^j#uu5JLOSFG*jCrKWLZd00(yRFS=f(l4 zfkM6|O)HO=1r^jqTD}G2ril2sRIVh2GSJ~r-A!dPEtR<%K7&RUDGD`W##959q66|^ z2|2^zNZJ9$h=!X{!>O4X<%9O6v-%2*K)gA4uvth6_o3io#?|k3-M2OY*Bz=7*>U33 zEl`c>x_@n8Ln6P50SsLZIhQ>Tu?DNeCsKyEYoR7W%mX<2oW zzF_C9TzkcwJdxcw@^5~}-?YbBR1na{JjWGsB`G-CYg?nCElnl4%~iaku5*Swsu+dn z9s9nkeIA5~>gw+@0fk;~-aaiih6;R^E7+jrQ#i8C7=eH5%jcb-7wWr7rwK-~9j}u|grmm++knQa-@ zP?MluGkv_@Kt~ZIl#_YII(agbuSC(##cSnzSt=WHiY_tZv1`6D3@VId@~&*kLT`&3 z%-Hz|lp)z?@o95U3ELq-*LD9KG#rAo{R7>c&;nR1E|M1I9m@bOvQGz`R+l%Gq7TjH zdy?-v#@M(@ExIEgw|GoqS1xM;|Kw`?-*3Rlg^Rnz$6#* zl_$$T6jfzA#3bj;SbnPF`!q5z#_?Qp&+%+bWN908~M~JICjB-!Lini6S>L#vT>SKTVd26H^&{mD%yHFG!5XVJ#(2z<(Jl_F7N5WnAWY4KL%4W{ zE>A0YO(6fzH9k{xjXpq#Y>bV^@Ook(<9))5xNBNIU*n1!8W(-BEj>#>OWhHInWWx~Ijq7sbEla{IWCEm@@m~L3fV-NX-vHMTx zZN-ZAys#9k#1uQ2|N7zuWYN`BVq{frq^oLBSw$&W31HhiDlOz~_i%{=yv*z>$%PX3l6teC0@&1{yxrM>oQm*H!gNv z?8?jnbH3()c)~QtgRid13qvC~q&w`IHExf9{F&HyXjB_x+}#q`ecR)cyBG8hq30Z` z`r~hS;wsz*CZoT|hl5FcCnrrqlzNFGv3U$72}^;TPrRV{P%N9dydr@`8t!FtLDe$U zL>sFa$SUu!^+2}ffhF{)c}L)FYpYKoK)ay{sn%!&)B<=9F-o(>7Vq-&PLK!ifLAbX z(md)S4;3CWZFQ=q+Ew%X_sq!0?+z69v59~SvEZwC#{VvMe!SFX-Foi9Ie9YlIa^U( z>n3jHN?E|rrZ~#x{EhN{I;D|giVCSWz+)E*le%VJ)+f$ayd2~WVkmk}2d|&Yg=iWo zj~%XdQHHo)v?fl|QNp8G8F&~q3JlpqYQVxB+uh>vqf#0%^4RgI=Cd+zL7)-LeaW7i zfAhPl`oV;Iec=dS(M%p1+7&I0$vA_2Qqm*d{70Cw?j&fd(IQP z|0LemcA*Mr#yuXw9?_IwFg)G15l|Bbc-+Xjq-AsZ%7!<|VGn&CkK^{)cyvJX{An19 zq2rKu7dyt)#cRbpxpcAbrjdzN@~a_lD$VME_fDY$uPokKr!|c3FnOvC@IP|Ai)@>4 zOXe}RQeOId+v7V(q`A*&dMd?Z<&@~rdsu;)8)r_$lkw2erhYKu5a?%NJ@|O zl|*>$@kr2z{>68!kY+Dl!7@}W5p0NYY0>h;(G0DhA0vR6T3`!jkHS5~iiu z5K5){04+((yNhuc%SqADuHaoMS@^PC5iF>xQh*bCpGb3Pn-7!<(-29qTYQ^N^C$xf z27HzMv`DU8FHzD1IM8^LHH73me_sJmcC1ILT?d-^nd_sFBmd>g6~$Ru&$yY#Zsq#3 zjvO;af6n{bCkh}3@;i&hEB&{HnIn%?iYCvVl|Lq2O%*qdfAhP1LFKWI+<+aK;scDoH+f!$KiDBZ+mS0&s1t;@ZViMnQ~k{ud1skpQdog5Z{Bd= zSVa^aEOKz+k?+L9@yVS>JYR$)Dtih;+O{+n)*=cH^1cY&=8Hji`L0+ePdjQHXn{Wic;w%BLCoK7Zt8jhNYG>T zCn-Zg~D939?XgSYUi1ENNH0u?f zNu@zr)SsZYoj%M@%UR{`X!l@T#_kIS;?$HZRy_yvM`*Lk-)dIX>L=D6FT2Rcqs>P` z7G}Z!-B%c6AbaE9zeRpxCcKB%Psuj>vE+gi?gkrRl6W0|9aUY?fK(g<0UvlF{MM54L^eLCl z$;14#{BONCsODG7#S~MJS4Bhhr)5(A+MwhCjLOTU24e|zk<2EUdG%?2k{_rI9}*&l z8pYQa!ptFVMhrYSFe&z6O~xZgl*w$w9_d;kXB23{+sy4-%5yCqfrkBq&(wKcy@KWH z`zjT=7-b9~V{H7Vc>x|>TrGzJUnME`7BevKq-~l`^VOCBV~{FK^4+pG6l2kn4Ic?w3}y) zd7?SAKINTO1tjsYd8BG2!D7$5BRrlo(pC6ujhEIRK-BE>(qR;mmpAG;wj}af28XTz&JuQv{fuEg2SO%a9)?Vo@bM9D#Bj)ZDA_1rES8b@ z8xtG}_dD+>#&|3TwoVwM`s=>rbWK4OX$5Ayd&4ug)}%$fI4jq(=#rxfRT24NP-4?5erlc$NEf2BX%Gw4R$9ve(dmRgh{uK zmOt1lXefIi;+EVbiieU7gfUL;!cz(HbVrxaC@{W?3Vx91Ytg$gKsqs71<%Ip>XYRA zyA<9#@+xDrJ7)5JQbt%Pp^foX8IpIkdngyUETPG-(sFl>IX0WN*S2Nv`5iaF`B)T9 z!1yEAa#Wg@oilRtXBYhxLi}xKHw**K1H$ImB#@_SQ6MX3UFwB8Wl9zgu9K&WLcuH~ zI~3%c7&@Aye~mHKCRmt5o=dz}3WFuB2rKfukk^v;iuDcaj+gV`Mx>!&BA%+ZLkb=* zaw?|_XcLw@@L+_v>C*hB>GmWTd8Wt&ns%oRXQ}3d0}2C$XFP(EPEV-)I=kKQYDX@O*d3E?l@-My}vBg5V@*EFbMa)Z;Skd zwGi*EMq^n6b8TPE*D1KU@~mZ=29e^jxwt2V%G_90Fko;{p^KN!iCMq$6!|{hs4`@Y zE{}{VV|0f(9s^NZtvWR&+o{kDT@w$shaRy3#VJL`D@mxDB85=cjrO5!@{_7+8cl++ ze=cRT0u^X>z+>xJLdAUcO27eE)FfjNF?Ky)rMuSEx})ae#C@&o`Nl zs|ESD3Jd@^1NPWfds+f%Sfqkb#ZBYi{I2fv^a{or|4M8Uxb{@}k;HfHUdk5+jz(&$ z=$KM57XnX%&#g;;I!f}@AXzkhr<&DKwUjV^RfZ9qh>c%GIV9xa`FI51e!BeY)vQGtwyL4)V03p;?@Ng zXA~ep*M8xe`ZfxiXXFJaH)Orj+<#J3+R)FobT?l$3;WkFel7QDr+Iek6M4RqwkzWu zZxTdbio9X2;gHcNhHROUm%L@8{FCoEX&!KS=7kCB1o?C!SsV%zb}7P!$hn2Wm(a_! z@sej@b}^qgl##q4p9o_DPm}cEMXMWHk4E1(jhJwX^3}49VgSiZ)-myPTfumg(#i;Z zd@CUclv(I2-pMqVtse>FM{xY{a&D{wo`$E@e0-tn{)PaLT8v!Wj*Chl46e~NkFgGn zGpO?Q(Ph<1AndAcefJm+yog-dNH81zd*73z`ykRnpSn;!AtJ`cKD${<_0HGJQ)b=z zK->MvAgUEjBa9n6&;@Jc8?obDG1srP=$Hg~bpXpiG{1QH^4>cta?e>UUl+v!r^zKe zxK18x;c~K|-xr;>QvS$HE?>fsJvou*wcLl+uxX|II!W8H=TKA}(IwE_jP*HH5y^#f zp>8SKbYL`4OeB6IKjB{C$+HRW!Fxf-!)H_#RU6lglmm}@w0|Jqc+na;91BZma=Wf* z|6M!+i~%U09Iv0x6BE;-CX6&nejfU%f>J;aQl?%u}B_W2jpUFCdFI z9-O*PLZ~+%1}vDDo|XpYB{pdxN}>+5kJE^U?|zVXYPk#RzyQ5s!tPe-CFuB=TPgdQ zYh&{V{Cu%%q@p@@oCZ=1Ca40JcSlYj?ZP$kkXalGlyu3B-rrW6Nz|OV|b(} z6+D?3VMI!p4>;%5&+5t-g1JA|H=05wZytJ`*m2l!Y{;eeOfY6ZhfkGvw6UGMBl7!- z7C<~hEU2;!!@mwJXrZtfc^4TuxpXJW@A3u)9<;WMW$1WfV+Vr`ubt-**{XBz_`;Kx zzt-$E7a4)%`rW1AXCz!%%guNei80E*@@ zYUIr@DxprQa?MbqTpy9XWFtmVOwspQKSJN-x{=z8^@ELKEXQ_K66Xne0`o%r!5;<; zFRQKCIQN)_Aeco_dXNK$vGKeUbQW7 zM@*dwPDZJDUdD^Yyc!#Kc;&RmWZYyoL`Cf;GuBtHkmnbDUn9Q2oL7~Cgrd6GoMjEn z!uf~XPha>pqr4G)PR_gT>w~tMc!Dtj1tUe3nRmCRKA<89W~Ye()Ixu}AcW@QIU^jk z+w3rFI1LN_cH5S`uo#)K3QL-jEITb!KwX2DH8iz==7n6l{35|bTgTYZ7*w=8Mb4U) zvJXDN(090xp2;*-w()s4Lvm=E!Wf#_K-^dKeKrB-%*r2On3CUFh4O-D8Vi53dC}0Y z;Hf066}z6M=xJDtoa`4H0otd}kY8M~R{jbpxVq0b!BH4MDdT(Y6h7=o68NxpO0Lw(n@e4b{F_RgJ4T_Hg+%PfrW`CL?O2%EXHdoRDxo4N%9rF?VqOok3HS#o@XToz8IE1F$lzJTkjh>! z&xC$%nbs%SYsN^4ye*Z~VN=z~p{h&rxazEUouC*P17<_=NH!0b@k2p@8Sl3=bzbzM zF))js{lgVfT{Ip%JC_uu8nGD(Rkz{H7W@v9kSDu_NWT$Apuy`U(i z$Y8-h9yRYThr~?W-wr81U<3pTBq$VkKRKotsZ_W@9TB2^(9qi)9)^z#6UMnW`31tY z&kd`VKa%ew23~Aovc(%w)a(Hy%>UnRJWc+^cdVD!Anaa*eF{T7RTNn<$wwMfr0*v4 zI&I6eSE_7)$9(s+e4eJ27R>|0DWtVcBh6?qCBLHR_XilCG&Vn*61luepYq4WtLdBT z`2F*hd3+NT!>oP5q9&UwxoipKdyU;Q^5riq9$k>aVu}$U9S?l+1T)Y zo$Z7L4XOmZjZ7-^l{5$V5Az>hyFxCE750=FSEO#l7CLeO^Ipghc;=`bWH&&JL5h^h zgs00ORA7`H13Q*_e+MV6lB;Nt<2KQ+2}TDBJrvGm-|0{8l%Jb-?|Tcga$!TK4H{S} zMbm65pwJocYQD&9X!CO&|Htz<@lK6|B;@^btpO?0c7RvIO#U~L1U_m~ZRb@23864TPAr}@yfM~eQfQ2{3{^$s`JNn-Ize?F6{#*iib)MGjit&7 zJ$Dsf)Caf8&sVIFy+sE&px}wZ<5u$}&k+(NPXkXLeXs=_wT}xzK(T7P*8~h?2ST=x z%+k-DGjfa1>&Tc_lDKNC;Dd?<_mryw&SGxYtbEwf;?$TUT$pxZo^e!D7(;yHDtRT} z=RUu}Y;Fk8^LvbHvzUHo`;1&&Lmwxo31+-scEct=wG)PphM2huFtEtu&NE+NeA=AC zK5)&Zo>6lj9^oC}r_C82Kcl2yI4_o-L*p|tWM3!*s$?!NvM(ay>~&Lfkd;33n;*xZJpoTEXeZ+83$A=ww0KUk&ZVo z4UNpoXW@}jd@Tv(|G9>7Da8iP!VM9ws>t_`^g}!ge-AJhqCxodPWcaVG3z>+Hn;SE zSqQYpOnCrLkZ<3Xp}lyxA`hSj7`x<$QH@PwJVq9Ax5z!lyhT9)##%NEX!?Jja-0pQ zzW0UQA+p~(DHf><0~S8_oGC~sE7N&#gs+Z`i=xnB6eNU$UwESYUxaY^xu8_fnqaJ< zVjHp1ucI~s%xCbOnmP2xt}+_3a$}@FX4c_K)IYVr7$sl$6VK%Lp>6ULyn@tvx z8e04oYh3&)AYC{vSR-Gh5mIYvx9GuKQDFg%u$qq^q)#BfFZL2p^;h)W6)WU?O}LVK zVLrZ?|2|2$7$Ah98D|)ye{8QVav?`c3lBRQme9mtJ4S5Yl=n z5x0*milM?D?{(yrKpNoh+G7j?Ha^+V8k-~xJ5MS=>Y*(1dg6&;L&JxlTne!RI8Ww| zMBWfeNuc3Xl74PB#s=HWSRz+7JNNO-;m!-a6FF?ZNfAXci|SuBdNC6mlOV+)Huf$q z76%CCq7!}Vq=IgZyw$v&GGi0AYYH&pqE!8{v1#_2BifRm^7^E`4S1e zgmMnf$rITLD3$Rsi-cJSez#_cU2o|P96UiUp;jGRC< zuPTR-{GR+5sVwGR6}?2)Zjc}Ku>{}r`6I?y`A3==zkf6935x@0T)21H>%$w*{W@cH z*u!8pX|I0If?sYeQs~kdw$IAzLwe#E`T2@+j0K;!j>e{y95Ft-V^*$XY~;h)pEyH) zarUe{AVc}QD1Vye5FIrq5bm(+ok0!xc*~p5&JZJgr|@i-BHBcyt489@d3Ia4w9>#y z!rF!U3=N}#aUqipM(+_TXiE*bg0&CZB{XNz$WxcL$ADRw?vA}oaEUDp2WS*ju*Qyu z2Z+RVZ66)8cu@r{o+6t|6of6`_D(>*y;=UvT;nUXeQE%G4rv~RU1ovJ(<`__g_sxf zIjiOV5Fx5frp||O=3=3rrm9J+fQ22S{YMK$C@(Tu_O&C@d>JG@1)<1t>v$<;BjERp%fk7J9FYt7+0f{|7Ya~^! zpk|?_7z%G2ixG{@1>VRqo<)q%n7iMmL8L~m8RO?@Dywt+kO}=FcaGhsql$UeDf0a@ zha|hb$J~@$_6dbvD`6F}@1^!TWkP^wDWV{FotTTNTqlNjOahDzpp^6eaBUaarkYE; z#48kDTdk2DQuv_ZAzpvH>}lh=%D~J`_s8zRYdi%66O1=B8z9&Rpr2~BV>@T%!!dH= zk8YP=Xf<>xD2W{pZAfkZm=JsgD%lpPaGdvvTo*imS7`e-Dquhz4%O%SsCiik=78S>%=UfgNG2pBOP1Gkjjd z;GZ#S(mue|=zGs>m2tZ}Negmv*$80tKs6LGoaaKhq#5JrZ?fOfRO0O~QSlrr78#Y_ zO*8r9)jUy`7DF!doftW{pOJ?6Q~g*rV_ZkoJ~j=FQd1dHZo=kNXJ-v-3`h*kKF*nd zu(V=c_{F-ZW_U3t=nk3`h(o6q%Y!xw^6T~ma;up9&RJcG**Xb{f4|90_+_*O;V+7D zs?jcwklbo=6SJO%7N9apM=)p_K`fb@W{ssKZW+z7BEeF$OJ1VicOO zmg}o36zaw}Vcl5ODu&h6l{Z~QT0C?e9OF4TnYdUEDX9Xx zOL{>m7Qvv%PtuGzc;94%iMdYi7lx?M@HgY*$+GfhZGp`|%rQ{5(%Mo7-31K`4SE^r zulG!qyUN%-J(nIb!T19L?$A8&g-qJbO=& z2jTrAd`v?u*}UM7#;O_f(B}NaLVr+K>D3)%{_)oe}5!c_s)%dv1)MCGCqD5}7_ z7&!(T^>L- z!+8SYY~z05#U7=v4I;si_VxW;o8`wf5^nRn&a-WEO~)J=`xW{26o^QV`aD5>*qX?z zu0C0Qn4c$MVKJNJ&K*4iJQ6P*$Rh{m9_eZ|8hvBeQ@ zNIt1PkC^AGZ5|#N!0(O}HPDLaP`U&j$ylFRh)VE!slfu_^OR9ovCC3_RKUC@6iH6t zn&Z-(t01`cn32r+BvW7l`FF@8Owo)hu+bK7om+wTuL@8i$?_c*HJ;myCohXlA(Tkj zA|F@1>?1{IlSx_t_tE8R;$eY6x^Hwhb>#qR3<$iKQmt^Qao`1^z%5HV#zuuXt|tD| zRAZO1L^zqMN{TMd?@4&v%ugSQ6|z~a-U*f|aAYt?p^ipFH#k6V}J4nF%CaXq+A?lo`^IPH>1B z+%5hic(ob0neQ@ z5Gc{uSi()YjN}IKKyQ|wdZH&d8W>O)*;dPfnmsnVyce&P2luU&2aM5k|HedqWrX9= zzl&@eYtl?|n-0#&lSKwq>e-x;FL&L?Jn%%tidQVf$ymMocz{EN71q=I*^-A%cFxEx z7q6AWG(u4|slAX-<;_{d+eU?X4d+q8ywg#SBz>w=|2Tj&3`n??5X+0!$~Vkr=bWzl3V8110Mq7HrQYkY2h?E9ainFC z%Sia0Xidig3ljv8;6cNahDQz08Xh(ZIO^Dq5V|Gw@#S1~B{?8w7X+=sF*i}=fkFu$ zAzm81HZ-0GhJ*C<{t(LZ8B2t<`zXn#!I?AirO)gkC2=@$e?)#UOJFRZSGnf z2GzW6bv+L}O7G@5Q+bSJ+JZ3>YC3kQeuUB)Qo|%b1yd6Sp(;~yBHQ$eHU+AwKe+Aa z%@+o^HFA5oY4m~lg7D|d*2!PfAcEgeLf_p%Bj;yRQznK;{TS?zNIDYo?n5EY1Qt8Y zLie%l&sJjeRzwerqW!l5R&QRNg2z0@GY8emnhM56-$0QEMiYe%G#1|-^=(pGgPE_sQWQzRV^o&M(QH8Oiy44W z&6xG3_o^x52F7EI$ezuidETGtx~~&hXe8k9?yS|Z(;tP4*2>po$4B8~>$JShcMK-A zOXuV$rB2a*g0g(pfSNaJnr5+5AeW^>}l14 z`&qQeQ2AtCVQt~ zy@=2n8eto1H zu)GkZF|piy-Y}e*?0)yE_3~8Ki{dCqs*)@B30dAamus2wkCm??SA$$g^o>K$#OCXj z2FfLbRkD;4x1S+D!waXNyGUj4d~_6qzR_x~(yt6^NomkR!;wd;Sc$yZwz1A3EP=F; z2BVtB0$;bc`ii-DMf1KB%aK~sXWr}lahTJ{myPj5lK1|i6XiQpQuA)oSc7mnY@2Yk zc`-2n#}JneFP;m#5a#4;NA@Zhj!!*^gF@%P^m5N|Rj$R=M-^N(a?F)X?i-}k8n-S)GibxhF3B4%@ zGivmbJ*px}h_sky6XX)7?OR*FhU(=C1JLs0y!;sDD?@~w+ zBY8hCs70D`aOl)xWv6UAI9JUJT>*)PWV?z-vQ6-u^PW|RrU`#HZG2f8n8If|p@Hcs z*d5^ID_WGO0EWtCl;iV#W2F7XO{dF`v&SZUyJuEz^Lr-T{;E^bl!96( zcTEUbD=+7O!5dqoX>dvUj$!^5+^t)K=@)Y)6eWlhn8nANM&KG+(tMsO=+L-(Pst(x zi{vb`Da6~rJcHMla}d5eGce!eL9T4tqD0~)A113JxDXF{8ryl}?rbcylm1KzaI zIOh4Gp*+GV%6I_xV5G#ZosG$ghPf=Of_WjUGLvt1!_sID>x2k;46$Kl8XEYT3&F6% zvxY*I!7>c#tX&e)d1&4k8H+P=2hHrlZ;2h7^T$G(ctJMwT=Q}iVAK><^%7Pu%2^BL z@nwn)j+zX$-a|7k9%<#m3lQ@xLEh}savn^{mA5OxO;+ZuRs#_4N+`E7ldq?S)}Re| z0NG&uMeO@!6(zJA32T_n`Md?$hVFEkd*eWLG*zY;KQ;tbYA(|hlwe%$353~uf~bBl zV~~&`pCxZDcHMZCPEX_wqjrp(J7aJbZdx<4ZA-=<=cO>@z??inBQ2#Mw{O66A)k+p z0M~>Oa(cL$SLKn!u#tW$9BqzytXWTs^v4^*L<(-X0gfRR7d4)k?9C$hDc=FM3`3k=IX}X( zrr)K!>+lHP9m)Bn;bp?JSi&f89m~Z4#6zjcwSz>N&mUBzED&O(h@+1FEzHV=cmnah zIVvKG|L`KsFU{3VtJd@H5zhbvH{)SS$-s;FAW1)146H%>*2cBK&A4bQBwY#^#&;>+ zy-5}B^1|kp@{ki`+iuyq)J0BC%`crGkGLHeVUs_Jhje~jd(th_aw!X4z8HkX?b3mK zpRizKS6m%)B=qBXpz5)V&Sw^y;sL^Vp0B6s5G>#{4G$Zi=Xv_hWfq1fN8zg*1%$z4 z6{|600dH!EB>3hn^30ABUJWQQ&N2ou1{`A)13FY4>O%R1NNIQ`prk?0t;#*YyA|3X zSOWK8fef1kh}iRc|0H0+c@-8M%2U_T>L=4DEYD*WoI3iHw9IM{o<8d-Cd{ zK2ZI9Q(&W6mY1+i;Mvopg|PYRy07C1xJ@ryv&8d!*9qk}?zI9+BbY53KM{%5KEn>< z%q)Bln~x1tA)U45LONza-DM`FdyTR4lv!k-G!x<9mK9KM3ucn~ee!9TsF*}CIIcTY z{y$qiTV@fP4|!)?t5T&I0{N8ezOxev*JAGZ6Qd9Vj|#1^5HN^AT5wRbh~SATGL77;?Y4~^8ev?aDw2I(R5N4zX)9DQSTS6k3>TCqEW%a$gy9%kfElw^ zCt5YesaCGqmrRTH%^0Txv$6iGd_EWV6>rzJ5<1)D>XEY-yU+XWeP&+WA7BK) zKnh9!3G6BGjH*Wo-L zF?0W+Q{^8<5Tsz<7}!>64l(og{q&>a`E@`x>E^9rY6V=+n-clV#}#>GU4?X-1~e?6 z^CVj1z1lr3pD%GZRmc;`x3c_kG2oHH3qYbi%~q#&9cEWm@sigTISwi(A0Y<~4;&4W zYh>C=^kweifWEagL|gNg+UIo_UNu_Ts@=N5t`q~b$lFQ+J@1M8?DI~LheBQp(j4$~ zc6BXX(ZJxv3VEx`rVWG)IvVH7hO!CE?prI5X2Y$vaPKK!o^6vbWBNBWsRdjoyt1*+ znfLSY(E0H0GY2}l9R6M8mFyX!4j4ASXH2KM@DnJ(x?Pmg=Gv=ImLHU?u1;VgFdN(d zhc%6MU@qlIWu7au&cfvIefm)HlFC6-j6qwRmG3G;I!{aY21Tu)g7b#R4wYSa8J$&4 zm8OT&uPWfybE*x!Lo9@w@hoXjiXSWsa&2PVmh`hr!=-7=oBPbhfaV#~b5tZUKGP)p zQ-?|TdCw6>@<9*ruB`z5TEmwT9RqC>L9?NYRij#Om)c{E$IwTr?<=+dc!Jqn(5xC% ztretf7FY~X?+6v@n9uxvEG$jCY)<}BV<^@ zz`XBKk1~A9+5?)+-Tzr+P-2{GWeUNGE98Z2s2eu`HDu8=Cive{xZk@9=`;zA2t6;L zPx7t^g!e)@FP!hJ8j3P7njqeV3J5MhR4K?+E0Yh`5)!Iqi|z@=bz24(i=py;_-Y&d zX%iHVU<{!b%%+frhm3cB^Q^p};44u>n{YY_|7@TfwK}pY)f=7*3H_46loVtj3KmvoZJ><2viKVSZ$c zca_6EuF$3)krSvn-r}r$_zMA=9yTv#oZPNC7L;4>vh;nUU1hA(X6L7wkT6Eb1q`Wo zkke9CS6GG<_s5a1J2)qg8bk6CODEIlOSN%EE+Mpd>6|=_@rq#v#YFZq8m>1x@sydg z)10-UoiD$ONquR6@#phgvanme!9F3u(95buuz;2Etd0uphRAW>$JQL1DLi!Q5@C;IJeJ> z4qsux8j&1`gt1v$FrM*p#fZqwLj2Yyt}Ene#lS64!D$4%hg3ULsK9zfK5HB2U2f2B z#$LXDSU?rSh=rX;!9@wHU!el)j+gU_h@!KNT#+q_ydgy0VnaG#uhNJ4X*rLo-~7X% zFok@hrgk9UB*=mc#(O5+0n71q;YPH?HaCmJ|y`>BR>IR z&k@hlCFTfI+6C|9RV!|@&~V)`MnW=dT_8i@^i_VAP2|VW4vlfH8U5d;`d)fI&Q(Dv6|*$&&Zb> zy0k%vw#*ry+Y|YXhR#1h%~-uc_QvWTXfM_|cIGMAiyiA-C8T%RD6wH0@iG-Bnvpwv zxfN+Ba`xGYA7c%Uz+6b@`!=LgC44f#$n)JWJdQ;K#~3acPlW9FNvNsJN%1s&dZ+w{ z@4Vy!Ax*|-Pg*6f;5tJOmdR%Xj~W%Ddu^+QSd(IbSiTICXp6+5wZa(XA0ec~_~bfR z_^CRziXoLfr+9y3oQW9DWyf2CNUPZL#SAP)yf=8oG{-8FWqKDyzjh5Zf$c&_Sb>|1M!{c=#}od9T@s@vd1C>mzFm1smjPQq4>uf6JUZ)k;DrA$mc5>s|DDsp~xZys|?3xNCB%>;B{h$ zoDK29>u0@sMTT8Aq_gDjcq8yAxG>(ObMlxk$rv|mh+B}xgpd)BhcNW36a~16hM)YO z--~VnNMe&PEn9WUBJ1-mWYivdlYmQQTkQI1P3Frqg-|jy3rFCkNl*>Jptlleh0hBf zF<+H--$Li1;u*%PCh-{QE0=H_YD5Y|bi3EKVKzM1Yq%sUhE`MHeIo2eZX3oXo~->t z>Be`IIS%9NU7Mek&djznlsK9fi+P!f)=r)iG$fjNCy+y%P?*C#rIDHtINfG>wy>?y zxmU}>r_C3epdTRb`7ImepGKKK9XcrNtQT#_wFAu;$=Gj=9kII(txT&P*_vhjL6DHwTq={6P}JeXZ1T8DsFR5KBe zVJ_RkwUNp#IGyrzDG*_kB#$ST#9w}rYv?oP781FJf(UkOgysnqq{TCt)PSZziU8>c zj;?|b?8eMHi4hwc$50n3sAE)|eWWKyL-~87LYvPMqX_YV_a7RPqD>&*4xFr z#XLeP$gWxWBdK7_5JII`2g8(nM?y_qXtd(3)66@TJ9TV8s)=TN@t$_w*9ppDaaLXj zEz)Ne<>$`xj8TX~Nf|*&Vdm=3geYY=?pZD6iFN`r*&OsG$`WE72|!`T(tJNH!D` zQb92W)K55O;FNe52+g10_{C*lHd1fQSNqXW8F7mzd59YNjB^#$@`>Nhy^D+;lACwU%7+`eT#B9+q=w=riUWMc5MC)8 zUM&qG26y#@fH)6A-?6IowCJKbJy&Uv4g(!e-cwK11$n(b%6w|TJWT)d_uENCj^lhf zL1Q4*%{g*fAqwvtOn|XTl3c)i(liTMUKi7~k8PKqI(Z@tGK~jNL&yxVtxtZ8W0)8H zQE6|NnsNdb?JhA+M%ght={Jl?JYCnEEI-KSl8qB)SIFL2D299HB+0a!hCUHHYm$k{ zNjzbtyzxB|$_;do&4%lc_S`3}lD&)%QXFVasbaW7F*YK2<9QePKfIhH9oLNO^Hazh@g2=yZ3ZH~c)xNdzEea#ZpEUFQVSs@p5H7GgJcK9x27= z@0p6>AYgST_lZP#JSj95RFpI73XRTNC61S^lfR}FEpjA=hIaAnq)9Lxu`8A3(fWp% zY#iQQvEg&=0;$Q9YY9mx{G0WrV3?p~>C1_XR77_MLdibL1PsCwl7GR|n0A3}qmKFB zhJZ^LfR#5mt5?VctEc1uO-kNNbX4)0QJ9q-msC=pX$+~VeVSl`33>*zvwMdx+1@o< z3GBmwibay5PfO&DQZU&-2Y{V7oT=(w`Yfh0Ld5XP$w5|NyIM^RCD)D&6 zs=}$nk3DB-y9hB?xa>3+&&;8R*#KgE!QH9-GP$DUL>Kjm`-tYSgble=fhQl~A(x$y z_arB9f@J}Soqe7xc7g#-R6K+i+13dwJ>g_3D)ACwB(}I{s%T~`&)^)njdYlULcdP@@Azso69=bn_N@h6RZR2|47|yJqFu{O3tsp!ngc_3{*r z&owEHRTblG0A?_X6kH=9*N%`sa|#=TL#N0O`F|BM|G|{}%Iy*9OU*Na3L|!m`wEHz zHP?!Pnh$+rjA;wzFw$cE+7rZ8Uk;ZkJUiWV^|QP`XCcAX*MO^ICY?v2^({iaM7TrkMAN~lZrjK9Z@ zL6!EJ<5DMq3bB+9jg%fymvVQl;dj*qEr;@n#aRlf3~V(sY7oUXCS!ZY4IsWToZ^Cln=fQ`k_Yuw{Z{19-K4 zbVfAGXW5(a!{+Sjh-WBP1VWJy#R=##JMSo=H^Q;VIu(5fW6>3Ql?fFdiy*YBlszjn zkQ^vZiW!pRCrSy!9b-S1SBUxvxz)a7Xmz6r^%wQ+jA{A**?ap~OR75G`4P{YtyocQ_($|Rzg0HaqNL1oZfKv2kt zLD4*1oIDRY8Xa45ui^x}0~v`JoN;oQ36jA$j?6zk>r;DGQ>RX?mwMY(d!PCyD>U7; zt5#L5TI;udFW>KNu0GJ3L@m{#MP~Tu^wN-XtGX{}{MxGB2z@|&tR}BpTvc&+OD2}c zS$X@y;)K_Z&@v4JIJsaEjG@n%1gqq|{uCX29ngZCU{lxnoo< zr}dRw*BN_6p?ki?14Ld|JnYGzBV#e5dLrpxo#RJa1{b*)Dv*;0&!lTyxrp+~nY0zC z5VFl0{w3GX)i35&oA+()m>j|L=sJ&yd10@o8j;IDZqcGDbScC*q>KsjoydDSbA>## za+%CzISGWyJ_CIK`8t!=c<^x14<8kUBFYatJM#XdMk{|ldb0d7BCG}&gog~1UyNuv zEi@PCzwb$F__c80u!s5%=>_D&*4@`Q(uknhG`VP)PE+%k%lkrp7sv7q+KL%k=`J(K zrL)oAFb7V@=$1^NKm{W(?r~%Oqd3J-L}-OiSefQfeIY_)r6Qu4$g_RaS7_=PHaU;A z#VDeyc3n(*Nf$n$T0Yt=`Fb0c%N6iH*#>!ywkhV(1eR%gZyS?)8VRbHgi@{2hxXizTfJ zS*;d}7d}#VB&-)nc>v_@g`1gac`*v5Qk-O}O0Ec96Lm+t!OOR4nf!?l!Md)*A6ZZP z`atHOsiEbVd%Cj|@>$YS@RpLVhcqBnxRrF)1Kh`bvvxHoDxk$kW6U$q7^ykeBi8V`PKM&e9nX?#T*d+@^k_|83SauIwaHWSt7Ymd$sOoY^ z5RmJLXMon`J^F*nV3+!iPB$n5YhhKGw>KfLD|7)((-Y;@YI>;!-;s||hrv5`g1r8N z9#l-^f>E_!r0-1xq1Pz)kqxgVIesAwvyX)eGD8cDE0aSYYhIz+lwYBq(=kHJw9_)A zNT zt6l%&Y6tWw8JwVZ;8JtI8xiH;ojxJoHRTC~$BoOaIr^~2N3jF?mnBP@slw1iP!Iqo zUz`SqKl67m_1E3MVSuebDk_dDa(+}FTpoD7&%EbHHHVE;JY1{wVt=9Lo%hC%^(g{C zu1p=;^~v*!!o@|NPt!nNBi_2S`yCOEx-Onb^qCnm+!Ny;&&=+Q+%m|HuyMgliY7N| zFl~fqG_fs^cc#g@#v3xEQs;MdnzB$`6CO<&azCi8lR06dyGXj+$zOq1g>Y4L3=;*5 zOaaQ8z(rpXEx0n=lk32MVPf;DmSMas5`>`%2%zuYA-^X)!DeOOD*1jHyHcOLTGVXF zjmmoNtrK#&O$jHhs@E0!2JG zEfyK}c@WsEx#vG?%?YH^&qgR5HTCg7P+KPp0FTgltL1Sfc?v4HSE_GM>+||82p5ri z$3#bCl32E^7!?LNP%d68Cf_30s2E|WRW0aGB8^6)zvN5R(HRUB*I4D4I{G%G=&cx& z!?dC~e=&G*TJk+=qUuagn^9*KGamP#SHZ@F4Z(Ju& z6EcUksfzCbJ~Y>R3Kbn36fP;4v{ltdjc-m8V?c9p-j5C}jq1=719^4w0_qvvOny#$ zSc&v>KkeP?+VY9I~&P^y5k@GqtEFR&T!qhQO6)8 z6bN5o{McO``J#>eUVB5{Yr0ki*y}?!%Q4bl5L#$ZkxRZwGA|F0*xXuqZetGpR~yvE z;!)RoV&f?p=K_i^d;Rdt(F zxd7IRTp9%#{IvlKfl-Xl#+s z>FN_+fI;yPwE0g#+!q$$woqXHmxBsyj`Sq>|`Fh^-y0|qN<9i!2u|l76 znPnqI)$yc39fM*Q42&c(HQxKPSBCj`rG0mJiM}u^zccj{Z_$<*VSSyzo9&|4;@_+E z!Xa6Q{u46hnGLQ(2!em|cA^sEcr~A14im zF-&9Ay?IwvC|G2FC@ddex6nexb;D`e*naAh@!roMt z-X9Cm@eacjh>1|TY^dR*bVE}MQ6s!&K0rw%$psmyjLbAAwIr~4|Du+bvcR0^G8}JE z-*z4v9UCSCc@q`Xb3J*I20?Mr+(#SmDkgP*wNCm!?lBl}=XlU-Xv5mIq2ZZN>XvCu zL7vX150KQyug8WxjenE1t%z6LfZaDuw>-24bSa2Cq{d#TShKFLShJ`LV>%sKXA?rF zq3|J|Zp$4V`Ls)fT^kXPMZZCE4(U15u`KkvC(8F!lmRt){;?~W`r1wMLRoYhJ`20v z35P_d%BX(Ebv${1qIZLMYizoxOxhKAHGL6Qv_@V7lL~|jc)M+(94a5DI^`Bf3>sTK zgy=CgAaAa_uNe_mvqxTAU654v<}lKv4ab>!M)zfigydd;ho*wC00@uUau08wmKV2- z%T>wWsd?{VALEPT43|Fcg@YKH#%6E(xZEb5Ynu8Y98I>~$ zun4(!?Qw&wC@%6?B*IB(g?x?U2}ik(#S<2tpP~RNN7L>J!=Bx&4Ee-CwW{l#l^~0j z7Z|jYQZU4hgdr&jwtK`oJei%6Wf}NF;xU(d*jK*o@=TJM!JM-}86Aa~gC&w#{*@4^fW0mp2#!^RWty?O5&M zZoIG`qB{8v>*Q&gZz%1Oweqy<9@E0ycyJCoB31l*nx_`p6T+>g-(l>NgXroz6`n5g zY;riGK0vkOLt#beTS_bppjgHBteTWBI|!^^BImi%(l8DSsDBxOcQ;ALL&NiXT1UQB z_VqFYyEuN!$cF2EtT)$2ad|2d{%tPk6vCugbQ@qqi7yzI^U%#hm9+J%y^13TeT3B7l z|MT~o4TWHH9_tI=gwM@m_!`rYs>M7j0tFV%DN^>9i~tO6SI1!_oVzRV9}4~&ke|%*Td6v zT!Ex-a4uND*tMRf{vA(EdFjTpn`J4Rr|L1s36(4J7w8-c4FHt&gFg&ik9C zOXXFrJ}}Pl?8L_b-z;Mq)e+P0sLC~kIvJC7)3VS%TO};_oW}4F!AC6y4m(q!OB7Cn zN{6|E|2n*nlQv^KYVZj97S~|p;=#qc2SGVw!azpt#jAR__ewfc*`UA&S~42^fK^?m z$=E$1_qjH9w6a6-YMpgD*9bjJo%1mjzVF?b?tO?FB+gkEpp6r9fc!JOEpW^6vA{i) z7RL42`Qo;g3ixwafN}k*iRnp5SJA)K`Fg%D#Y(u#q>sT?W}h+K$Xmg%h_1u+DK;kUrYNc+Ik&mqkC#>lEDwc_@+?S8O0$ zxlLu}kS+%JBAdG8cn zKG!p{6k>RJSv8lQ3%+@sQ`dQT{BO#aW4>3lbHt)RgmG%~KJkus*X;!waj@xiaiY@B zD0%JlNi|aI#ec3iq2@g7S!I-|j-*zyeqyt{q!yndLg+{{{TFe_bg7B*uT4A4oBEYu ze^Q_isZd7hYtyJ_SdgiF_O1NGL&fn)PavUM44bMvZG;Zb(4I9GjjEDZ^I*o|{>Cue zFeckLmcHA(3j7|`7|72MVUW|uwJkfJ$7BjArd3U}!xEWF~7Ci{(xt?cQ zm{Dy^o~X!B$PDHl3{9lu@ZwZ`Js;2?7X=l~=FtktlzV0md==!=K3>NPRb;xP9Heb- z!+Fi%9AKlcVoVOxJT?t2?C7d7Y4yZHR9__3CIw!_V9Z0OjLXpyubU4#srN|=I0&~Q z4|iaVJWr4RtKSs|T+1_`zd;n;G)XXpJ;pV{Eu@!zTM{oyI zKAo1ydn#U-!Fl-}eJN4Y2J>pZ9U2=*TZD@0U0a$C8#B(4{2k}qHZGsrIU)D)-@IuU zI=$r~!sA54XjBL5RGoQIExu32A*lka!!*3rcoLz|$#eY{_Ix~Y@v5k-C#3B9R&=V- zRN2?gi(ujeIiPkOM7ERYvsSG{h*Qd4}jx!DQ}}XM#5({yX0ga0wN=J-n#A z7tY7nXZH!Qavmb>CWCOR$He=BBqm;0r75{-IXuH7gB45U93QDjLhv4^V~)G-cy92x z+%6s|KB%SOQM+gRD+KOr+%S?XmAoMtbDHtW-=zo*5QcvcBP{aVMfsarz##7&Ov=HT zx~1J~@CSx;BL41tx6E7%3pNMxW^y%2N5inbYkRm^d8(>{IJLIi<(k`h-&I_@g##@3 zu8FKJ&Mh^OJ(5=G%PzL;e8MV#(R}v~w=wi*{Y7}MkFki6DAL>=uy1(m!bGJRMu&}h zghpt&Xdw(m52g*zO$PGaVa2R6XrWm!Jx}d@PSuX4LXwSF(mMrluB~`uks!Fx!SH~# zaM{GhF@)h%)3tf$mUq;J4u(x9G&3}!3Ta+AJbosRA%D-mS8;PFh}b-=nuoQtRG7sI zwUOBuh1aE8hl-Vd@U3cIH)(>G4ocGT?2Vrw7s+sVkt)T(uDcL`R(4{j3wZf#1u+lx zlZHP($Uf!3q%Hq1YEjANf9v`x3?Dr(K3D6L?yMHn}I)8j|ZV ze2W@qq$aqigLIKqkwTk5Q_egY>0cJ*F)RWkw$c`HCSz2G(kilJ;9jnYRj!ky-+Er8 zPcLH9WUfN1I1auYR|r~5?AhKGYu{Gg796)hjn|J(r_;Q|HcT&Vs9;?CSX%&ktunR+z|eMvV_EBA36ef0t;L#-07xyK;fjs0L|NX5~LwfrwE4sx;Pc% zeVQGYPqL^+c~?;`6=B4@{V4*{3rI77Z;Zy~vaY@q0Yw@tU_4W*qDduyjz#CuRc4$V z@F3l1jVn?#!{*wR%E7-IsJ4n=D_%+0{gDc@WJy?k++y&^r6CP)JdpB0%P@x(gEIc! za>Gxz1h7 zn%~)_uz_4V8irQ}t>RpmoRHVCVS_4*rVR39%Fz06R8rFh!71wFA>;WLA=mZQ`}NFo zoCge3o*`AzG&Y{G2*jk%+@F@a+M-f~sy7#Pc#gYCK!FZm~-;VN|awDXbI9x~-Ilqv8@mqebtb2&hbVjVX*^{j9UXYtxdkiaAW$ zH<6t)h6wTI=xZ%-DTCacAqP{*pbplJCyb4}2iy+lykO0?J8F{fsJ{%l+($s;M0GjW^BWu=cD#O*g5jtx%7K6PS8L$T@w%OwgMskEGQ`l&UdWwxmo<@O9l@g9@RPgMgY!X$;vNmdu>QJmVWy1A77>h)%>h5PM z*rE9BWbh8u-PZ=7cbs&!W~nT4P;8u4nmn{IY9CdD>esVY$fI~hc`h`Ab1_h{TN_dac?MpWl>a+zJSR-P{S zjC@g~3fLD`?{kbF*86;TY+cWg_rk$=RPRnng~9m4i&*g9jKBajDdB`3_mc9Y%LJR3 z^~>c_T8YJc$DuZ5kGbBP-f%R}IA*ux()gvFx4OEjtrocuPC#jt9ZWu27Oi;$t{PZn z_NZL-p+`X;&-EQ}4P?6paZ`1G#ACwhNe>Fe9~lYH=K=lHWX@w+OVZ!i;1jYA7fp-g zm@n{p5Qa-T{|8H`z!sAGr{q_zdx3Lw;sm+Savm>TS&JbV8c(Kg31a4yJiad8Dc3bW zI6#Uf6AvK9yM+fZ6L=11uaqaqlaJpg|IgoN$#?86lY_il*E5WJ$dxgE^!(NGBgmp@ z{MlE2FejY!K6t{=K(4S4-e9xBriF1w^nu3vha#tIukO!+;0h1J{f_#$g6%jdkT=}9 z-QM|QBOLY`W-!4pnK9u<6^!gO@aj6~D@F<~i8vNBJb|wBFnPMdlm)9`hXp;t@i5d! zYBeY-6g??My6xFg1Vg2_@A=N~FxVLvC4|GeHsWU*-Wxn;MK0o_OoE{$_R!^Q5krz^ zY9qeKh&oGqp<>wjK@XC47$zLXYw5_M1d{t2fxMey4{{IrFA<`w=YUSg$3xCI6Z0m_ zSNAvC z7x0q42c`56r$uRiG9;>CEW*(?Fz|umNh9n^E26I9!w*REySSf?HLSzJc>08VH%}j1 zh)~UK(m~F{4Tr%V_wb}O=w&9BcvV#!p%!@O;;ci>w->n%hLtM^Q!TFl%XJ>|u{^(r z1%{-L5sJ4-5{oC{-Gau;qK5X68*?b5#-=R!TZ~-1e~}s#8p>60Qf?oY+gz$xv_TZ0 zxPa%+G>_B5be57LIX)raqXlzLD+R_K)^gH|f>HQiH^}o_1DUJ4k1a+AB+VW@S$-Lx zOrr2i77vhyS9K|hBWefs2G_~(s^Y0i8W!XY>0M)81~%7?_@!#OK5}?bk&c#}%ls_J z$Aq_#YU(H~qrKK&%5y%OatQ?@HXb(3xA!2lAs9Z$C~|xu~IOZshz90ymqPVth;Y5Mh7z%v27ZhfSa?0SiN$dCk~BS`L{>-MmgBZ$%@Wzaez{ym zE65Gk!>~Z6F6x`+&HIXxBXbLK@p`IPV5DE~)F+E7#F`}LG40rQL6Z2d%<>iugm{9rLBN0Z&4 zR#Rwf7}$sE0$#r?=OjnXbMM;8%TOveP~>kXM&;$N2%LBHlds*`V`*4#;#Y2B#@MD0)8U_9yLdu^m(hIF3`5 zfsd9fjU2PAzo%%0f{i$>2=b74IcGcaUroO&!ZC-@d{)1D_=TtvpRgvR#s-74MFJRg zRA>`{U`%1eMHSe|ylx*dY79zA7#~!#M^sr^6=qKF$aigFV9VxWU|70TUWE*w>)E`u zFW^v?hm=kBC-}ZFk|}s&(>w_B^%ib)H!PPcHZGH^)(3J)-JK~BJtkZhb9NdmT*o~# z*+Z#=fR}4tPg>qa?A8gn+*VX<^@mAVHC;*>l zL+)M$m$qt1j|#ZSt~CD!15M!kGbVKStUkL{{*y&5Z|JH?pdi7t8@qM0yp&V1p-Swz z$s^I3O0y*y6-;J3a$7tM7&T}K8cV`y0SYeUTDhJTl5(bV5CV1Of@v8;ldxE1(M3NS4>Nrr-2g{K0ncCQmCG(t?N4kD6}&Q6YcFb$56Gf&`Fi>JyVSyfyBB(m zjX^WZn+XM_#pP+&d$Qp-708A`o8hsz*Htu-2Csa*yrh|-wAu>!cBDg!XG*=}w^VQ+ zl>_P$mzRomQCEI!ItLYluHQoeW#XrTvco5rhDuv?_-thlqky=28hi=All+03^!SPs zLE}*=i3qw;z-t7d^c&X6(=`4(%=4uUgk5n8FxGJr;4Hwf(FGTc-Prg@-bH+mj8ofY zPzwiGNu2eVc$Dvl@bl(z`G6F7K6r}HTp@o=E-rIEh9t#AWl{A|AeP1wz~;tQ9M&QQ zgf}i(E6=6T#oRnCFU?MbgP}1{JMurdb+i1&_Hi)aq`~=eHu!ik5@D=HfPA1bs4hkw zgsgoi7Y=vL6?e8mfcV632?KjoSd(%f6)co9fxL%>ITBZA^H4@k4GX(#3=-nfA=5Ne zA_doC@YoW_ADePqo2qOPf#NJ%^{M7RJtm$+-E-!u*gzSF-%HA!;b9{j>_f7jMC4IV zeutzoVGwQzicvR3Ab*xVG8fX=$Yd-sY@rbBUuEZ7YeMp896mBp7)V;u%>@>>Y26yJ zBlOV%rjbNmcVFY!Fecjw8JhZw*V2dTV$`w5lXTUla&eIdKNVgNn^vyc={t)8FzU;` ze^NK@O4+@$K)h~^{Ip2rF!PkHu!V-)RbF1ohNDul;2}fTM!Q49=-C>`pVr+`6R7e{ z%g4j8iYl+EicO=6Nj8%@4A(G4BXWd~!`$Rc*4m#tiX&1i3{O$U$0^Dnl?IW7Yt<%_ ze8;kk%SD5p8bF`*2nD^WBVVq&uMyPZ=UliLyo8A@rwPa{gC~rTF^%L>t%_2s=g_C} zReLe6ud|-i05ZO3e#cdv_G5c0%dra5Np|iRGEDjVjq@Dj3@`D;8XH^#Pgv7~$F%BU1 zmFDNVRR*+ST}0xpO;!VNe2t3Kpdz`ms3>XaQrY7}GMz@$jrxcB7Ojyq))`k}P#iw# zErjBW$l8&|3J+-7x!<&2e$l{!t>%K=xQ|{i+MIRwHNtDwxD6yt-C(Q^#qFK8yJGMb z(M2r-_U}t(79)Nez_3cvmE>g@xr@o<^xd&vLhz8&d zWn-ci)FQ=G0gEB|SqHoF{qxs^d8oA}oAZ6889KbrB)8mR0&}P<6@bK6VzQiEFH#X}v$xqh!+xPJSpe~^;odn4o#y3R*)B?KgO{edOP=sDG#p$DOX|l}=x}&i9;g_~=q+4O=_V)pT>u zut}$BF5Ww3u$#UB$}@5~BVJ%OI1o~s-mmwG4>)uRH7V_(vhJh|P~mme$@0sjCyP*T)W)XSI=UbIVuzI)5$-~jv87`T)zpo6I^UyM*@n!y%t=l6>x zyyp+7e5iztsIs6nLK(z#JSjxiY?2pjeZa$=MXd&-)x|DAC2&f$f0yr!^$(Axi4EXz zPo8j%dFbJ$mzP3X9gq%|=1CrOsu;$f$)k<;cbkWDlo3?UnN%Do5Sg8jTS~l_(hzB> z;#GL#I(b@aYT_2Py_7~7fVgxq>>xVEvZql%p$v?tnnq|%-WuAea7EGdE~v5IrV7zJ zsx}`!i*L8U;>+_M78yWake3LNx#^jOx~MC|kZKxwJd0@jF&4`%Qb~jA&o^A}n(tPB z4}*f2@(y9e^?}R{uwRV2BqBnkoEq*w8u>_D0(oCW_f;EwXjTRk0uJ_BLQ9C= z&t%92sS^F(rf1RaO04J->yvfq#@}-#!pt=7mOIjN#rf%yF&U61rW(O4^U~3 z{XkA5IeGv0Ci!=?;*;e4+@Hw+B|J`%Q~dZX4*hah`d1eCq^odD8P~GGCs)(KZ5k$< zev-j|!<4*ugBCM}CMCtyO7$I{uf|rG7KN(_|w^@?S7bCx) z&*F>0COIZhcHO4XRmfdd68+DHaXE`T)8rUrGNFj2N}+S^0cxNquLh5*X<;Tk!sciMTT=2_MpjPC3>z2xHHnDhGF#>PU1#VhlBm~!Rg`mmfOser<(BmCVGniQ~i|~lX zFpDSA^^Q13Do$59=)w6)WByD%fm%==eIzA^g$D)+v@F6rr6$n8_!to^j#bjVL}7Gt z3~A`kCO`f;51Ik!J*w(s69H4d%`MN^%2PN+@x%Rw9dXn!hNPfU1(W23mfYVMA79nq zK@^cNhPO$PaE(7ahgEeDuSJ@EZoILOO$;C$orI|p9WPFai(xL1rwu2*|*&?TKw$HWe#TjVg^HrO|Fy9eUcUTm-9uAcf zH?3QnVgT;*PzKZ9;?FGjG5LEY7i(njoNf`FA7P{tLF)C!dc7@22H)m8st}B?JuJi>nBY?|#Jz zDFr`1UZ%xL6`)a@6r4{UNGRuJspQIw8AmP!VKox=(Hl<7*Ge`P9T|Zlg^UO=&LJ3P zGN-Yj&^xhQ&*HF$$_-6zR30eOaUZ>DCUJJvMXpl zVTx!Nm^*h5w)n#YD_QWMLGgua6QeenF0%775fFYPd`PuvmFW$>AZH6NU-H>7u6TaK z=q2@`-=Sgrsk!AoAF}Sc)LO`mwxx>kZv!g)Lax{NJ?UbdZ>9aPdzDAE;_X}IWhCM9 z_HJV!ujfI~G=b}4H6}9w?LybPK`lZmc%CHWx~@%*mO6AAWd_%=7n{Y#7Ylm26?#`b zG$}eas1VQ9g=^el)jB8NIDB^E7>QD`;X_^%A=fik$UoE6eD^d9UX|AzM)=B;IXpY1 z0q-mxQlcq@%J_aLdeA8v16!-034)Xya`Nc9(6o@J2{j5089&IO#j|inZRD>e%uay| zEu;#|S2)<+-7f}%QRAmCHVMgBqdXE zHQO2P%20NQR4d|{WT=laB?3R|x3pcIuTP~zDU%^{kCw^~OE z3RU>B8A+1?{OGj&S{spGAXPxE**{GhZhUhXrCC%-@UHq8fbjWoJVuo1+m6Y))#*?Q zxp>)xvL4d-vj^SWJis@p?)BR^Q!wIaH}+LuzDK%*Z(p<&<*|#p^8NEx$@iv{3NH+- zT@$YmH~NHtsR+(;Ga6l+VL(VQ!;Lixgh%nbB^P20vb?!nCJ<&NsXuuwv|PzCE9*MR z%YHZ|^pI>!SQ976g;3Gh!f?pe4P&^XiEtHXgffLC$a`&4W89V7Nd;dMsy5k-QRHFj zCt=yCPIYq)GFs##c1+)JUjP# zs7uXN95rC!$08fi-_px6HN@AzaflK9qRACkFCr^?V+sQk;f| zc;ke;*6T`~@V#nR8d>dXpxkWJ2@K=MIlMQgO~_YWo{Iey@5-wH9As)@`kC!{!OC6c zHF^0|I-$3Gs3(u{!dalpyhai}iSFz^^_cf~v&^!>=%>Iu_{Xt=%=wJZN$9J{%{Suz?O zd_QSMlM7L(kMEX71>|HPZz}(8=YzB!o1p=zH=J*YpLI(915bK-%(CWU%@Os)>+3M;6lbs6idd0D)bXk*(9%H)Pg zbbjXfYvlW^fAo7>b$aAoR5KXY7}0|ul0RLIFPm31B)`{mO|u>OG?m54=NCGJ5}b&s zh)qaOM1QwyLLN?==;@TQz8{wy5Bi7)nKDGSBK3O^vS5!#V(Oq};;I1QDOXsHFi+xV z0U<#KBCX1ubBmC`_G6I@Ai062kXsRiI+yF@StvSKEFajZjj9Rx^Rlm122L55qtW=& z;pwWwLz6CPD_SJ`$3*UXtZxUNFVe57Z15a`n2Su664Oh7{IAxjZK)UxUk<7K;Lvz?ZR=EFnkv)<{o zz}r~IiGtTKZGBSbN>&InUafExOe}r_kW_w!BWq`4Gp&H+K;`nbK!_+5-`HB!!$e*tjeaqi#5S zlH-$frjLkTG%S_)cRI*8VuTi{H|o*S$N@0ERfhA>a&_Qukmmt9v;^w zH>+`z1sM#`?`JtLMI=c4xz3A`EspiwhmVk!wO}XYmsLhORr1&yc*R2{knwtI1F(Ng zxu9uPw1l2X#YuXQG3W{Bc2m@fOqcfI6J(A#Yo*&zq)5klwdg2rl!aEVwL?MX_Or;JCtabi_y)bk|vdv2ZYu5DETnBCmNoHkg|ho^#C!YQyvl$KrKy9m^tucgW}Bq+NIz5RX@J<7yGVh&*7|^;z7-P=U>T z;vtNRVeqyTQJJ7p)#h1@^XK)J$w8O#o;AFgv9F)^@vfb>TK*>Esl6O9c&%I~@(Gjc z^)9b676xdPJ69JbUB$3$RAQSFYhI+*EB8&sAsxb5h>MZ6@2*7FvcdA%bTm}LsSTmHUPtl+mZ`W2rV9`@fnw^%Q|lZ z(2KPeEpp-2E9IDNA&=+q_{cVq10^2n0jNNv@x&!tg?hhUn%8p!q=52~P>Vkgif(MV z>gV^RHY?Fe##85_IblBEl|#$gH73JiF%P4SsFG`g!+6a}Ei)*DRqA;8X8KZd^WNvF zfxPoUk5D`${2V{MT&{72?ul-~k?YerLdULaBgyIGDZE2%$WjTPBSZ%h?k(^@ouI?6UP#fvZq|K*vk&%nFX zWDuHXoYJZ;7K%VvjkE{y5_Rs~qA{xR#$RUf>|NZGpG97I7IT}2iU!HA&vjQUk#l^) z&)2!U0tJ9!>za#{p>b~&*G_?#=PMp!XETjACt1iszo9;XuBXEO2CjmLOM3E5o`?>= zxx^sRsW&9vPJ*q0{Hdu@8I>Dy=Q7{bwt*R7rEJvnMIc+XWS6ec%J$%{U_ zkEVhuUsKi2P3lA0lH}eg7hYi`R75K5!_WeQ$VC0X-_H!lV5$O4`cYZ{3mEOzE&WPt z&-)6MOQ=GnO?rm;`Ia%FxgOs-1`dTIc{~@x3iaHMkQ#w;5v5CzIzeS_f+x+IqsDX0 zV06?z7hpgsL&+SC$Y5xcfPlr_^*kt$z<7@Tjw(~COIhThM$<-3C7!=H_-0KUdul8^ zpQ8f!I4GilM-I|$JX{oTvEjvl$!b(m$&I2>#8pU+A^DUjr{wKl)qsB!<0)+n#8oGh>3LvrDqhGm*sNfQC>kBx4LcB3+kaR^Y% zbfbgw6vHdxWMs~B{t=ds2G)CT!trR?Ji&T5p3PO~wgyyEGTVw?Z<*s|% z*JsUN045$Al?jth8j&}Xr!90rgCJZ@#=EP30>cO9h)DKneIRpr&JlUwOdf$EN(uB2 z10F9AIaIFmu;@mN1a)nqP}R6)vw1Pso}YW_=vy6tZs*p`3oEtxJ1^e2Aq8XB<<;Z+ z*sVD?5mm6wpYb`6`SM1G!ALPikzytjrqpH&Ba!QKe>C1Do5z`E=V2}$5N^ZclH8dy zz)(jiIyw4O{t?C~>wZQBc%NO*jTR;Q4ikJ7eN0|qWYTQE)0LA8$Z+xOyLQ4%P--&1 zWgrisI6}QsXg_GX^V47r6w}%RN3I{?IU3#%yo@=O5(2md*%a{p@MaJ~4CTot zv|h2VU_Oi>oaG7&Xphy<<*oVfZjw8?>IAtU{yPjp zJVvhHrlk}xdAfebB;53ZFPA*qdt%a=!wQT;Jc2AxY!;9p3q6F1@H|G6jaZt-Z)FHP zqjdo%05mL+e@D*@;{Yzm0eVm;T{&j2VW)dk+9S0K1G>Gx}0NMpZ$Pywco{J zhT3U5rJgVlcb2eAK%Ie|EftqE@&J6fhZZC7iV&K&y&L|081rP$^QcTqlYW};6lW*= zm%JvqlCHc%w0Tnq=NknobzS$*F?IJdFQDYYXiIuT_hm@TT~m=r z)Z+4JbDl~^J13g$)vXJ&<8r%eQv@%h3}7tbadf>)tx(e>blF_s$jDn56@f7gr)H9+ zAj<1oBvPgU#L!`JmW?2~2_Tsf!$>U}4?!L}jGh=DeMfS!PN_-rutk{X@yB>euRm2@ zpo*3M^8dpwZ%26P+D-C8(jX%(fRq8#@5mRu&5|r0a?=^8U?4$eGEFXeBuHZ%4gsak zUU=bL-z>+dV~0w%5~vN$H7??5(jIhG?xRU_{(9V=oa` zyRPSC1oJM25VNZ7(f=I@Ks~^uez96CX!};l_ooASi|co6xF`mR8;`mO;umTnd8gy& zEK;}N{XW!_$JAu|15Kqf3?4HtEnkk2;kci9*z2(&rBOA>@o2iba+r~yTe>v-ZdxSZ zlZ#8i!1i&sdIswvQgq4972oBNo;>R!40rLyndqJB0F*SQjS5n8CQeOqSmEX4_ZVY? za2lZ^Lh%Rhf%%f}g^WoP~A&|Td*k$M3tag)U{}HOx^v=17A7}Xm2&j5jT>8u!}q zd>+0g7V(G-u_7|4L?JOIX}SVd6MVAvthxH_!c8W?2Hxipo@Ow=t*Hky)^_m8OWQE6||j4 zGZuJBv7H#mt0U5J{#?T2C=EVr0!gD8;%3vhj?29HV#AnhBMs=2o99<<6OzW8#xW3` zV{qE~3I8NiGG#xbEMVhq8iRHAD;4UY|9`HRkAb))koVQy*Bt0gE|)j!gw{*y?o5jo zEz*G(0x7PCVxD{c%-`*psCPmy0$urT4RJ&9lca-D=(337(IR*1{54@Jfh!eYWPwB< zMCv93IwF{c0_I|i+$3bwvmOi(q-6N;*bO0P=>m}7=(+HpU}6An%iHYXct3agjM ze$OrhDCQU|w`cm5+uJ;7#As{+Dtuf{*qBJ_>{ludb>+vd2TE(JW?Qrff%oMC-G;6x zq2UM!>oI}TbfLr0d%N&lK|hijsNlev44X#0lG zeY|3{1FtAq3@v;LXSA0(Oa;Rh@HyTm&l2DJpHC=?NV~Rw%UmEod zRTRnXvrsveeXYXq@c=&3R-LtI5fWJ+Su0PGPH^Y;K8M#RR4=!v9@FD;6fVP$&B$*U zKU6jI|7}W&8<3|{LFrnj@Vvx+)}`7sTI~0(kViLUApd8*bJ59Y$4Kscx2Ax?BbT@H zFSf{UBafbs!AOe=n_R$PbDfL#KQk`>!&M-qulsQF8l;4gmlwZJjMSqi%RkRf$hBoY zPqj{b!er2vd7V;#RsV=%pi`YVHa0qynk`yn1sr}R`M>vUlz(Zm5IB@aQDXHFRR@nM z@EV_^+TOZ5GAv+?@(KN2(kqWsPpx-b-?qW7zWmTb^9FMQonojswwg? z-KW+fKkn%y=hF2qG*{(4J{^4HI`g%o#{1tqE)Tjk0vPQPrBVd7$$Dp;;Uw88T-i7w zuPyM5=K-(cA%n-dtZS5sF#c|;0wxbfS;$|kyOS+i)CD|zHyJ{BQR9P`gj8GInCi&6 z@Oa%7OJ?VL#*4YLBX6s_e=S=0fE;cL`CLWmdhK9scRZkI&$#A7$>9iH{&?Rf- z={KyCr>VHs24m85Xq9=xTW2G_PlEU33PEE`NxyQtyLYAh@SYv=A4!A!{MGXJ>lPA1 zZ@vCh`IT$zT$Uo!Srm{{!%av&D(5fLLtP2fpn3;6fi(3}9%dlb-KSBUF>h5QY7Cunl)`Pa3JhYbM#+?>7I2Px(;~Twpll)7L zg2v#~aXGqmT&~lG*TD2^C&%S93Sg4=jU4Hqzo)tdD;6yQK+p4W-7S*>{(fk{L6BUAEXK?ag8(GJ)1V7@GH4Hp6^YzReH zc#Rgr0=TMd;W-j=Q(W0CY677nSBQcf8>$uS?yCkNvs5Pfc{}Tdg{@l{foEe%w}E+| znh3QyzMAUk|EnkeMpK5~KTnOztrGM8;0-bcOt#%&xsEY+m* z=ABq|Glb$km9l6Wm(|Y_5pd5KlQcs5>z2#Dq9?`CVtW=l#Ey?6cwSP7NsGaF;#s}a2&>!)~dKh ziy;FNb*>Ne%_qsvrI4%Fpg?zFtC2)gM^x<KB^vi#CCzHXV3mj+Ej zOpaulzA9dfgB9+=hY@&@?%FQDHN8JJ9#w{|nux;=H4yEI|He5MC5HGsnB6?^Ikm2+xJIXC**P)mm>XQ;mX;kk%N!-hvElWB;I^q&z9Ph zJ&9qLw)u@WLh3BgdHMoPimM--U+kgOCPvw~?yC~x|c`Tc8wJE=gP&_FvG`w(H|9PMu^A$^E zABsP%0oY<54y>^!rP40y=BMbmSWSYi^Q%ehAIiL^7L7%ZipMCi8m|JHm^sc9?CQuD z$U!44$pQ<(G~-k)xbJoS*vBiw;?yb*g$M|ZyPg-`R9~{)VF#NIS78V3=t+IPI2%tL zV;2ul&aQZI66yl0OB25hi$>trBg{%8l&pFAQjL>c6Y_Nv{S6*slNVDPLZ63wj*SB2 ziL{W(m|_wAqo20>;2Em(+@JGV6$Z~M&G4sRa04ccz<^xrG{cv_t0D3gV2HK`V2gRc z8^X@S^wFsS|Ld(Cd3(;&PqI^TO;8_h7uyyYVB=xKv*+R!^VyW7g|ikZS_3cvymd&g zxt_^fufK67Cl^(Sc*+4r137NwRu<_Cn`c))(ma3|b7=}17$<2878(J$ig>*^j?gaO z35Ep)F`iux($4Lp0Zc>tH|sMh19%=?6fQc{9^M;l4K}VwiZSpG4)Pu#f{q)cgS>W& z27_|Ays8t>L zfKc2S9eEge9~w#BYnRFyMbGO)0?fx!#yzwceW(T0DgJ){to+`V6X+8H{z!jQk}EH3 z)X)}zECgQW8>Y;%UWYA(M7`y54&G(jh8S;N9-ugK`Fe8|FnuJ-6aP&U4f>{sfR*8S z%=OG71t#W=8$F6E>rnha=>(>Oqy6{0=n=BCrzPpKLu#_X_OL+hB=5B)LE~vD$gaLv zz+`-$Y8R-sx8APQhr=GU8m@%`FCvx4w%=h9a1 z@tW20x2}nmv6j^9OoGi_Qad!7#EReL_Btcbi7hh2ea`l@=o6xB=K5lh+sMPi9a=fpOYI8knph;6L&R z0nhbX{c|VOhqDfvkZ54fmPZ~O*c26_c41992O1F4?$K>%5mkX*x=x;@sDyTTKXLfJ zCi2!b39#v#!f)5BMK~GW8Ff(o>zar$kUJi{l`JTDYY3C!mC=+A51%|HI|gwtLyA@k zE(Zva+xYPKH}d%g(HEZ(@X?dQ5aGbzZ&Qh&vlf96`TDvg@&yU>u44K1aikX33_WJY z}&YT#|$N2;fM=SfWURg$hs5Ap21PNX(B6 zT4R!ai#|50Nyk-1P0p&+7e*L3P(DCq!SpN=0c6ZXQ|k}s4Gt^+v<~|D|i}dIU-+Ut02Wyj4o>4^CLX(N;(a9Lj7dPff*~@ z6N`FUU+&p1FYoM=NVZr6AfLqL?IOR@hx=}npeDW}c<+-Q6`sYptK`YntK6btnEi-H zPS{^;k>4&^GMM~cayrv;v-zeLqqLtdvJgQl1Q=QrLUKIKWTZ((gICk|EO209Txt^g z@cgqAa&4L&#c_fP1GxvY9XXo+kyL{NYvc*n2eWUbe5b%Ay1YEsH4H-4RUu#jv-Jm_ zv&|)FJGN*(NI!6mPg-0{vuZ#Av0_XP)9TdB<=W)sK&9|UYvq5d@ASu8`$V*j+|agI z)bLtst{rSqK$tU#K|=ZiO>1F)X_4o2`eI7_(;%3WQq@*49M9~Byr0LQyhew-ZPIYA zs(j5ccpsiW+qjeH4;)mt#7Y7iuJs@gE=nIlMV2iugQ_>RQ~FF zg)yi;wWtqjBKW3jEDVYoVZ)egBaex+1U%<7a$=wkA}m|=^;@(UW*qdWURTE06E$=V zgQQJ?r~w#%gpRWoz2KB`jipQ1%QNIN+m4a=|D3m4el+L@V8V(qIdZp0c=pfw4ClGM z5bSggb>*jNFQ5*&7Kk{p9=Fgq@vag7HT=UqwYoxfWCKJ7c<@AjFM z16s6bp|CBGcighs90vZ>7Ws9ZRj%a3!^Nj1<4Q3E(GOPh#*fH5qw!~SyURHD2p})* zP4}or6*q-&uU;k3cpHFt0i`)RfDP~U&^RoEK$x663NEhQBria*L4(_=jtbT76LL!$ zUCLrN05?v^Yxk;mL`al|p)8A_hy)2KTfqa3lmV&O_`>sNGeq@fspWEP(ZYh=6LM#o zgdESkoN1x#tyiuV1@+$HHh9)JsL> z??wT>#Mjs%iJx{|)yOm=iRZ4ApL-jCf4NnDKXF1aR`N38$^kr9e%U-O@CyN&F=ftl ztHrp0x`GeU^>X(PFz2gos1M)~Dxu@|PXa53(M$^2>ZpN&hbvE#=ab$yQWxgd%5x~V zRNRFC`X1MqB+whauDL>GhM1E_d@VAf34qbAU2@RHTt&#Dp0^r3OfxX1V% zRI)Zzi9VZ>51%UkCjCuRqX3~Z75}sW7|ShM)CP#*Z=RMHP2VX079)>$h9D_4nC-~> zv-ZQsK&e3ow^1l!0Pa@n;zKE_JW0;Oo`(rddyC&qTCbhHU2H*!zm0va3pz6YCU5gx8@k!7J4ATxA3KI`^g|bVrS6Ox{o3{fxlo$yQ9IWw)|= zHiiZ4{4db0aUuB@mEp`Zm5MaOi(@yB%LmekptEkU({RaJ`3Ki|p(&9OcjUN`mj)&Q zzV|bAvvEjvBmUMEkxb^EMI_|SeWD|(jDl~{(CQnOdnfD*VJFgcSn#;+k$~CSk+;{~ zx2n)xCI@|_`qH$TH;l_!rc!wp|2R{63qO!kl4b`>Z{h&u`N1lR1 z7y)b>FgEs7CTD&yBZwFg^e=shpcgIF$ME+3tK^3;|I_xalt)htKCxMS za93Di|Flmr2Gzy3!UFy+&YXeXuPw@Q^kn(RoB|QS9~e?F_`CC0hl~MDp`2?98mE2S zD5$~uC|bD50Y(0zErh*kT#nePiNmT|b<7~|#TW43smb799j^4}WN`990|mzIO@^wr za57sdKa|AoSnX`Leb^%h<-nTI3%DpiwLB9)#T*#T7z%`KG3G{4R?t~IMEl5;yoeX1 z?#`BhogMkiS1eV57!uHp)ZNb(Rs=*QOv+Sv>(mD93fl=n-q4{%%dp64Rq~J`BwW$d$GGOO~+-aNOF!0pJF_-pR6i!@= z0Sn=v@5u**bX=w4c(W*QsJpLK0S_&Ce(_1_rYU)e1ucyNW+ssL_~f>XS_serYeB?Y zPLyAaT8I!`VAuO6PpR&HmIWVasD=^!w@83+uWR1IlPiunYx9J>4#f>Nco>s3{>qGBaU$Z%mHdmEtK*F#r|?aWQ;>E_E3#27648L@+OQ*8hsa{I{fd*4R+KSUp|)Rns?h6>m)_OFs398Y_L#%h^dIOu-Y1~dw9 z-|Ak_4b`Ck#fQX}W29e2(wk3=|ue$yE>RbbG>+V-#Gr4=2jsZT zS(17gH2Y$Mro}5j@gW%s=6T`m8s(hHBrtqjLHf}C`!n;xVR@z($LdgY-%8EZd?-y8pG0VB8D1xz0<-) zXoj}Dm(2$T8Q-Dl8|6L1V_U}glHMj0#?_|Khg|w#vs8c@2_FwHYdTTCrAuYcjZ@|g zUDU2PF=POl_YQgQtaoU|yH`9$_Q@A}xzzQ^3RlSDG@M9(&Ci z`CA5qI9@Ua2^Orn$QN-tVz?D}X7ez!LJl#x5w7~W8kDs6Zj|ShVf3)4Aw(tVb+78k zmuXsNr5&^_Mjm|hOcu$B;nvOan*s$9RLxIH1FPp4j z(T4(-I=pLAvWmC4o;(@92F9a{@j-hH&D1l56xGi7@4ud~{PCD!U3NOMF70?oe+R9}zQ`UJ~v>0WCUS#zxQWZ=iG@M5bWc_m4w^vIS zVN_C3!y>h*p=9bwsKxcO56qf(*Edhgi)wTD)&X>-OK3J3w!Le+{8l~RAGV40E%IW0 zAafK4x(o=uU9t1f1-7s>jZI}M>*mI0H9xvU9rmx1pNO`V_O6s4xIDPz_r3hT}+~a7~g2R#&eQW0T<`6m&+)Jlmp$4}5d? zF7qJ6Cwh>UTN*VCMBjpv#ST(!(w-%T-ka9XuSMwg3qVt)P)#|UaND}xIdjXl3Hk4- z9aIzehmxMZWV>^_{Ql(N6++R8rLx;qrK@iGH`5IcZ7_gw(MH7r&_$dY$UDk9|E`Wa zc+1TEq)uKUnx%^||Ek4MqIbwq5r{0|po@}b$K_Ty)R>pp-I5~0_q%mME>9jyrLzw& zXw@n)5HkOzBgcn(8AZyP7K4GybC+eKvwK25e{CX#1WrIIRkw_ulw_rVis!;L@^{yt zD!)RG5-;qTE9B8MKE>&d?8j+z#7V-KrfJ7Tn?s);(KB|^`k?#~<8d^ySYVgAc!Q6gEdOl0rkb?A>zb5k%2534 z=r=w_W4PN`++ z&2uvpPkd~~EcZs6>&Y`RzevT482fN9JFNwC*SV|YCvm0$8pS3$clvlvAH^O%&ys~U z&$DSUnut6?ADNPu9^BPm8{e~0o+k${O&6UsgF91-oo#(ZR4+mE!l;eat2LCqX7YQE~azg z`t)=+X1_&L`4IH^;k4fT7g`7<8nFI_B$5B zNc?*uBkO!#Y;sVYq(<{Ip$2xrH;ElV?SDq zC@}D#0W*yi3Wp?R4N)7Vw?GCR*OJ|oGKZhuk?+Q(-lE1P+J`f+#lV2~&S>8dO2!C$ zAhr>W(#@Pv*o}d_-jyRmzF%kQE4sP8Wpa_LRGio~`GvM?qiy#{$^eWvSjojO5KkVH z9ZULIQ4Ehrpi7v8McB0nhzN|>M1Tx zu#m@U+)Or#@%Q0V<=@hNtLBhjtAD@V_nv%swbFF5^Dd+|%mngomt5stY93IVrz<#K z&6x*XN&0ZV<)Hs)F`{ts`sLiXgkbHpNoR8IR_&u+V6?NA;_=0)L=K@^(|$CazIhL3 z#^nR&t&;zjM)bzJ;8acEe`P8JF}V=c;CR%O2Z%=xB@b~-s&EauCUm?7D^jZb)-98n zi+l33H18pO-f2(?@kx|r@NeytspdiN=*jMolj{55=RCg&0sL+lYtf_;V*|g})n}U< znUSWKe&j&z$h}to>*) zvhd~g%eBdc$Io?c_3b`W_D!|$1IC|;eE}|4j}#|zIj3}UZf*lY!B;861)~n53eOx$ z9J?pvbEXRrzJ+!1yy6X(CCiI>+kZ2B;_n#nq&<6%%9_} zUU(hox*RgRKk`G9KP2LZ=N9s3JcdN%F6zopbTrmJzwl#>Eh=%foRD*1LG`k4rF@5C zlJi!{ceAK%9O}xCU4xLYVw{2o;NiG=+U?;`lkb8YL@JqVu5TI-->3g3xHKLykR{=n zBBzbTfjlR|s#)}FPf_o{R&`7xjBm=%<9a@eSv=RvCmg3Irz$*f0U=D+_1}Dw{9G2{ zqPo)~-;8!~f>qvxjT7=(yl12>VDvGmpwQ#0WcMu=bL9|5;F}~)SI~{DeD&%grv>= zt>Cxh%H6IT)vTgriKJRQ)`>lKn4SBj46P@e08*h}SQd%ZfhB zG@xqXx<1ELaqXsqJk+~VDgl!%ZG+^%wDc2^r+p44#OyP!vh;(T!ZZpe#y6ASpeTSg z1#+=#fhOFH$8iw-X)$teDss^B5}5mhfT<2H`=(mBp*DqHlu>X1#(zh_rfuW@sV(yB zy6!0z6mi&cgvixN3lI4kfJY1-4lHG=U0Fyl%xEl_R3`42$XEPf0-ilWVYHNL+OU$) z^wQuvi-%oJAe85`(BhFv`@Ij}8v_!(%BbHy>I#J8R-GUhxcWc2Ob(Fj%2=R5qod!u z(R+vyZSw;96giAA0vCZZ9^E|`>u^lvHp~(rW2PewSK{dcxTtszlg~dWuBRFk^JjAyVZQ}SZ$@|IhOp6wI zKsEn8CSJdBxjp44rJpbkY;rz1w1}YrhJw-k=jB<9EGN5M^(F-NL+z}t)ptu_G z#N+j#*`8S*u#t?Ym+;z@H7~Yk0gwk&o8ce5VVyip3)6@u_!!ud$*`Q#qbhawf(qkyd)dAo^RoWJMaZNDcckO@L_Tb33koo7^8 zh@~-*c-MHDp^d>HWZoD=@dKNTFc(*CVlYB!C?&9Vij1mzbPnz8#Qci=6bwvakU8UZYs|Tj z$X=0$Jcz!O0iHV{^<-Wci@K(fX|ag0dqO_%${kDOa<^zOU_F4Rl=+`@9t=D+@BUze zAzjU@2qf-8Op}L614fpKV~(bI?NSR20E-h|tgF_Qvy+cn7Y3&1(5ua`ez>wRvXh#>F10EH0&q zQOEWPxyAPFMMkoz!nPNjJdAL&ck#CNWN zjo|r?t@4{$hrC=-;8mOslOu*NL+S~MqvU8%C^egI4C5!b+et&Ta$7Re1YbjQ)B@dD%#E%q8D}LsX>RM zeaVzf6hCFd$7`zzz)RP;&WNpiLp~dfi4)|)n)~Vm9GtFuVa}Qfj-@FJ31YifdFbL&{ma1@tdZ}yj-}FN!}9X0 zdrzN`@0tu%jKUNSzf!?RiAM}i4qlcxR2iPeRHq4MeIBAlF%Q3GML0G+r(M!`Z zhshjQFSQu$HJhriucg3;xR-|BaNT`vQ5&qhvYxC8ErtvTSEzaLQFsAq6y=GjCL=x| z@a(xb=75i6Tbk;|`amwZ(?icN3d3*?*RjXWC_mSQ#l&RLL`@gEEP5a(XEKgrM*7Q| zj>QMT?6}-YngL^lh9{9CgF#V*B;e#k*^~M7*+JMY@jLZx)*4EP94uw z+}C2nK=kG8mGVTARGa3_b$1L9CH&aT{K{|U zM)*}z*Qj}uU)N9Sg}giR?<_n1-_1_QEy>^Y#^ms4x7rOW8dbdN@ZdFTx=w5*jsWA`PMMt9f3_lxj(hVSoaNxjLcI6_`a)cxa^Io7^CB}mf$o+JYbEoMr zRtY|2wMlwDnJdcw^Y^~SC{#eEclQ;8>P`|q5`1$#d8YVXFN90|1@C11S>VaN?iBe~ z)A6JqY#W#NRMFS6!FT5>7(04WD4SV#=)FY)FcZkTDfYl4#DAwtUNaX1$Z>~7AUS8U zX}JUtq@%u;^dsx`bv4)L}{OqVpm2;lC;>qI-A%-^DFgw%I7TAd7$ zmnzUbdb0epZ7$S~iusWpLa6MD_vD!TrpM(dJbP69()hE>8>c4O|HtMr3M+djLUQV` zV%Wxj4gZyomm(_;dnC4#d#H(Y7eMv@!z_HJm!Nt0vV62X!lD=>v?lG|$ba+n09IL* z8jPPHf4E{y4*O^gXMwK{qZ<3)@;M&{75$`%n~RZu->REwyaci44Ijf@)5$FGZ+i zHF2`vh$5>jW41+euqBW`KG>bl`-rE0#HB790oHm~K@Ibz-t5)VKOf}}xOV6L5Wc>( zOJ(N(`x>0RQXWq;8n}U^g*rJdXF?nEU?L2Mv>Qq!RdmpYj>J;g*mEVDvmjv5`SgXw zm3*f7^TNwYDfd)o@C6Y`be70I)H!VTOQ@C~a75Q-V^Yh#Ld1{cgO2T*}FK16j?&aT6_MqXF*8GR1?Dp|ky2iLR+`YcB zh}$!gZ``{=9<8`wXV#>K5^sa7>Sgl(!tBDRNYpR_tML9K1E$kV9%LNW=!Ixe5!9w- z_oeI2>q^MC88m}$a})u+nU5m?uH8EN$K2zJmeR;stfPNPba!>+4wAPC;nLKC^n{kB zO9O<)A5shZH)$+NzNFO{w-~u|J$aIL-YPjZ?MRGvY0_fA?J^|E`&!~Y9wVqnY>OQE z2u+ymc^0P1)SMnqFHr4yZ%P^h4a$#_=z4;E4%a`PsCAYkU|X)*H~QR{aeAMS3o`9v z=2)uJOSmU5I+=!(Zs+y@R)j8;RJOh_hR$%yIJRi90MKc<3=kv^UFTuFAgY1JF9szA zZfw??%remgpupj?VSde;Q1W!(=q{6kcX-rS)YU%|FbYD|j+sE-LkJm92Mvjnje)$r z$jJ#M!)=%(l13X?O~l`^{dU{9yf>LEM5Cfrm(HgeG~_M%>=e?fq?548Rz(Z+-jWs+ zEu;4(`t?U^<$qt)l^^RIM<2s2pr8QnR=j`Yh+;fd;q{;**v0FH#;lZ(1dp~g3A9P0 zjQ9Mu4f6bEgA`AMb643@3hvn<|AD8MV^6af(={9eG$i*rzax4y=(MVi?+fb7~;(a4oEQVX|7{&}JkJ zs- z`mPUm<#F{aALkiOS}OPmttB;gd2ZL8A}>TKE56n6jGO4ZSQ26 z_JR*_b7&Z-K9Em|*Ufb-<4aOJn>#!5HpsG*JABkj(u_HbMLe|kB&vaL-7LRR&w>p~ zrf*|wN8XOdjfIFjNAmA@zE!HFzymnflV_4VfP{yWJb(aCYno@DLW(r*LzmWZs9dDJ zH;i8H+x6}atnOEf<5572-El{Ap8KfF-kh- zon8>Vm282uc`| zmwBe-1Le7YcAr%Jrd=CKl8V(*e3U?h`(i_6dPaC?lGG{c`?$}QT&?-`rYQ2T9mjZ! zihf+-8pbYpM-@J^76SmqTrSTrV?i97EguqdTe^e#_HQ>)2F2TfhUJJg4e*3gfJwd` zx&EYm=ql`isX-;s)h#}L0_HJPwP=&eEJbjHoZ@=BQT?ga*$Y>NAO-T0@_-xpJ#Dz2wpen{%*N2)HYcCoh8G=K7t^i7Y(?fG5#a z`3}`oo#7GSGuAIjkX9e?AcbckAipaN!5EBS#v43h>XgCiC9?l7w!EVCZG6u?d7`FA z)uJRE?8;-V_l#%GMPIeK&$s!!?fcqd$wl7fS;e}5CZt6wa2%~)E|+%4!a9Y4hgH>F z^F9oIoB=eddo>eQk@ zMYk7L6XTJprow7zwIShnH3{F$b>i=kkUBYF(YFfQf$MqE*u+ZH za&}r|f=fn{rujBM<~o=9PTrps7vRrFPnLg-ipWfu4CGBN9y?lFrQUp?@10gN!8nRG zncgQe4FoD@Ppf_~c;z-6u_V2lpbVDJecdt1;_0m7rpkm`G(VFh*=2E=>&Y|m>Qfdi zA6_(!4T*&heds`Jq}O$V`%jZZ<3k|Cn?}t-2(Ct|@PuSK&v{vNqWJoBAXnr#M<#?MyB+xwVM1CORfKP~DWL*& z_;mC;!dACg1{6(KE;_l?sQQg`k9E;W<@{?=MJIhE@&CwL`3ELQQ|0n$p2kWA(r#g% znpzN^qMx*|`ZV41h4Ra1;)Oiqnb;^od=?ru2xDV1Tk?HjRNwDQlIr5vJh#6oTQF{T z9%)7sX$`(bN*8-5{OJ73mR5=BiGWII!sfKH2(QWce8{7^8Y8xh>lK7Il@G-{E<=U< z(<(aYLw?QzMG=aN6OrSRmD`LzQ-KNXArDF?fbs7t=tv>cwQ7KBt>X`6eP-N~DPJ&p zSHAR1Et(7FEt|0q2dGc@HwnIXQyPIjXDYo6>mFkq^(G1!qDuWdeP}Uoc=18djQToCTWmkR0`j>DKEOy}Y~(-t@Jppidj2mn*9?8rb zO;QTt`p;<%V&{pavdeNBqF)!NcD}TEro)0j%@;p+&4rm#wrnSgSC|e0%1}FV{c^dC z)-(<*m*;|z%^4ZAi81ujLRuI*`%)4)wSfJDIt=_xgGxtO;hN?N;_o!Kn+-j@ZqtFh z({})o|K+O2&Ek2o6~5Rc__Dx8S%PFUdDJ_|`0u{jPpu;DygFymN zAJ;#axblVxc&nr8+UY)tI^QSnexGiYO#WJD^ow)oj4GoIelrZFb=ng#;aU9f#`%Kf zJ;<$Hzo8H#OQ90Em&{!(l7qTDj!Okx-wu8{RV^UrTyUI3D*l(rcyOX@h`A(x-&pq|lHA z9z0Jgev)u#JoJ=o7=Z@PIu`t`x>aSM2R@6luOypqex8G>fagCgo*KZK&*?JgVPrrX z8Tjx>c?hM*R6DCeF?K@qmM#Mq@Vw(wSOsE`851Q!7^Em?u>Q{bY%m&%KdEQoZ? zGbi)6Xpsdz%ndjTC=%J(kuRIlf_&sqTQmuuV8u@W&svUxwbWsBMrj|2V zFp0S&+b{ylO4M%}@Lu^)tRy@aM);_d;FUUYnH+WHtMSt7nJoJbpd4%n;|#V znV0=yvGqK>%p!(Nxrd+eq^LY4WQ`AB`--Kz+@o*%_S|V z*0q;<%jKMu%l!ZCoxN)mK^VpdEX;a|dRZ+L5fX^sry!)U5d#Sb#>z$!L<>jI09g-8!kM5N^UIQT5!wm{p!fX7Vp zT^d4K4BnYJ^G?!lES&GD@r4n+lIl@5b0>8qf=!0hrO7Bqh)Ai|H5c&vrcEe)fo|ql zG*{Dc-2|llV|vo03wVB!ugExOPd@}5Qx~;EIKgGgBA=G3=^A(3h;_B5o82n6G(ony zP+fD&X8Fn>k?JP&0;exrx9;+u@V;nj1*=H!X<8vtU!K>jU2QbN6M6&BVN?uN#;X-h z&zT3B;_WKUUbKZT4$KEeA-gv5a;D_mGA~kwtr700peExmK=mAf+hr-|>0TH9bu%L37mB zA!~s=MqA_ubU%4Xg#aTEi+HmUYBJwFNO6a>CqDJfGuNh~F>IbGplugrM}U$+>`t~0 zebQD<=nHffzhsV~8pF?mlhUmp8ik&+L?ZzJDQXCPjTzULC;Sh{51S2*X7A0R`O4qJ z-FGLiFZQnE+mdiDGtb^cqHvDQ{K!Pn2@x0_3h0UAG3a`1Q~Ov~$mtwu>r52|`We(3 zv3riQADy5+ zFuN+jRwAekx8S6tsHg<1t7)Xbb#@tvm z-!F{`toL+z9ux=AD0QzX?Ct7+$k|E9t0GX=Q2S6^4zaWHR783oQq40ErB%ajiq|y` z{gYD-WSJ5Yx<#~hwrb<0Y3jqqr0(I_1pz5*@3vnD!Lp!yXP2z*KgjZw8b|m+HUIzs M07*qoM6N<$f+~=qUjP6A literal 0 HcmV?d00001 From 91ce3a5489683454eac2a04046a5f719be8abb26 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 29 Aug 2017 13:05:03 +0000 Subject: [PATCH 384/402] Going green: allow to use bbox param, along lon, lat, and zoom The `bbox` param was removed from the base controller, the rest kept working as they are declared in the base, but it's better to declare them here as well. Fixes #735. --- lib/cartodb/controllers/named_maps.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index bde3827d..b6aa8bdf 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -32,7 +32,8 @@ NamedMapsController.prototype.register = function(app) { this.tile.bind(this)); app.get(app.base_url_mapconfig + - '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']), + '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, + allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), this.staticMap.bind(this)); }; From ca6eb609b2e5f08e55d24af3f6ef19420d13df77 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 29 Aug 2017 13:08:14 +0000 Subject: [PATCH 385/402] Update news --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 79a2bdd4..01b40345 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.12.6 Released 2017-mm-dd +Bugfixes: + - Bounding box parameter ignored in static named maps #735. ## 3.12.5 From e1990fc2f95cd8afa505b53b3f352efa34f54d26 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 29 Aug 2017 13:29:39 +0000 Subject: [PATCH 386/402] Use the correct fixture image --- test/acceptance/named_maps_static_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index 9a216451..b7af4197 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -217,7 +217,7 @@ describe('named maps static view', function() { } getStaticMap({ bbox: '0,45,90,45' }, function(err, img) { assert.ok(!err); - assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done); + assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), IMAGE_TOLERANCE, done); }); }); }); From 77f1aa7e0c845e01d22d6f4adf10d56d2bbf61e5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 31 Aug 2017 10:12:58 +0200 Subject: [PATCH 387/402] Update to camshaft 0.58.0 (#737) --- NEWS.md | 3 ++- package.json | 6 ++++-- yarn.lock | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 79a2bdd4..673eb60a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,9 @@ # Changelog ## 3.12.6 -Released 2017-mm-dd +Released 2017-08-31 + - Upgrades camshaft to [0.58.0](https://github.com/CartoDB/camshaft/releases/tag/0.58.0). ## 3.12.5 diff --git a/package.json b/package.json index 0a5d3621..a52a5f74 100644 --- a/package.json +++ b/package.json @@ -17,11 +17,13 @@ "Simon Tokumine ", "Javi Santana ", "Sandro Santilli ", - "Carlos Matallín " + "Carlos Matallín ", + "Daniel Garcia Aubert ", + "Mario de Frutos " ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.57.0", + "camshaft": "0.58.0", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index e9c60d7c..475b24ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.57.0: - version "0.57.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.57.0.tgz#22c2ab92903e43d5af6b763870839fbcfe82c650" +camshaft@0.58.0: + version "0.58.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.0.tgz#c352fd1f8fb84a85f580b84df79e9dfb556a0c9c" dependencies: async "^1.5.2" bunyan "1.8.1" From 5332fd3baa7602b228aeb6a398d01f589c129c4d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 31 Aug 2017 10:21:00 +0200 Subject: [PATCH 388/402] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 673eb60a..c37852a1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.12.7 +Released 2017-mm-dd + + ## 3.12.6 Released 2017-08-31 diff --git a/package.json b/package.json index a52a5f74..3321a45b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.6", + "version": "3.12.7", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 7be5361433c290e86fc881259708f74872cdd38e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 1 Sep 2017 11:31:06 +0200 Subject: [PATCH 389/402] Upgrade to camshaft 0.58.1 (#739) --- NEWS.md | 4 +++- package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index c37852a1..62e7129c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,9 @@ # Changelog ## 3.12.7 -Released 2017-mm-dd +Released 2017-09-01 + + - Upgrades camshaft to [0.58.1](https://github.com/CartoDB/camshaft/releases/tag/0.58.1). ## 3.12.6 diff --git a/package.json b/package.json index 3321a45b..058d0841 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.58.0", + "camshaft": "0.58.1", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index 475b24ad..8270d25a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.58.0: - version "0.58.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.0.tgz#c352fd1f8fb84a85f580b84df79e9dfb556a0c9c" +camshaft@0.58.1: + version "0.58.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.1.tgz#e4156580683f624212ea3020e59790ad006f24cc" dependencies: async "^1.5.2" bunyan "1.8.1" From bd8d147a7d511cacd162d1777cc8aea708312d33 Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Wed, 6 Sep 2017 16:21:01 +0200 Subject: [PATCH 390/402] Fix out of range bug in date histograms --- lib/cartodb/models/dataview/histogram.js | 4 ++-- test/acceptance/dataviews/histogram.js | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c27c696f..6e62ad9d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -198,8 +198,8 @@ var dateBasicsQueryTpl = dot.template([ var dateOverrideBasicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._end}}) AS __cdb_max_val,', - ' min({{=it._start}}) AS __cdb_min_val,', + ' max({{=it._end}})::float AS __cdb_max_val,', + ' min({{=it._start}})::float AS __cdb_min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(', ' date_trunc(', diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 3baf002b..c9e16489 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -332,6 +332,21 @@ describe('histogram-dataview for date column type', function() { }); }); + it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) { + var params = { + start: -2145916800, + end: 1009843199 + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + + done(); + }); + }); it('should return same histogram ' + test.desc, function (done) { var params = { From d77075295e1609ac1efed61dd255280d8a4bca69 Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Thu, 7 Sep 2017 10:48:21 +0200 Subject: [PATCH 391/402] Update version and NEWS --- NEWS.md | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 62e7129c..45a7e000 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.12.8 +Released 2017-mm-dd + +Bug fixes: +- Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962) + ## 3.12.7 Released 2017-09-01 diff --git a/package.json b/package.json index 058d0841..020438ee 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.7", + "version": "3.12.8", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 806c13beac1eb54700baf06d13e36b86b1833511 Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Thu, 7 Sep 2017 10:51:23 +0200 Subject: [PATCH 392/402] Release 3.12.8 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 45a7e000..a5436d6e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.8 -Released 2017-mm-dd +Released 2017-09-07 Bug fixes: - Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962) From 7dd5c5b15d178ead67609e797e949fcb1479380b Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 14:39:25 +0200 Subject: [PATCH 393/402] Do not use distinct when calculating quantiles --- lib/cartodb/backends/turbo-carto-postgres-datasource.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/turbo-carto-postgres-datasource.js b/lib/cartodb/backends/turbo-carto-postgres-datasource.js index 65109325..b2550eb7 100644 --- a/lib/cartodb/backends/turbo-carto-postgres-datasource.js +++ b/lib/cartodb/backends/turbo-carto-postgres-datasource.js @@ -21,10 +21,10 @@ function createTemplate(method) { } var methods = { - quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles', + quantiles: 'CDB_QuantileBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as quantiles', equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal', - jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks', - headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails' + jenks: 'CDB_JenksBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as jenks', + headtails: 'CDB_HeadsTailsBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as headtails' }; var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) { From f1246cb060178ce3abccc036e7d9f8ce20f5af91 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 15:53:59 +0200 Subject: [PATCH 394/402] Bump to 3.12.9 --- NEWS.md | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index a5436d6e..54ca402c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.12.9 +Released 2017-mm-dd + +Bug fixes: +- Do not use distinct when calculating quantiles. #743 + ## 3.12.8 Released 2017-09-07 diff --git a/package.json b/package.json index 020438ee..02e0d5ca 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.8", + "version": "3.12.9", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From a24792f46d140479cf051ab79fee22ad9a5007ec Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 15:57:11 +0200 Subject: [PATCH 395/402] Release 3.12.9 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 54ca402c..d2cbc809 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.9 -Released 2017-mm-dd +Released 2017-09-07 Bug fixes: - Do not use distinct when calculating quantiles. #743 From 855e5c9e4cb9d78b9a9eaa4ceb6564df4cc5e0d3 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 15:59:32 +0200 Subject: [PATCH 396/402] Stub next release --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index d2cbc809..15720ece 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 3.12.10 +Released 2017-mm-dd + ## 3.12.9 Released 2017-09-07 From 6b2e2b2241dea41d39cff82e52b124af17041336 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 7 Sep 2017 16:00:27 +0200 Subject: [PATCH 397/402] Stub next release (package.json) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 02e0d5ca..12430490 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.9", + "version": "3.12.10", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From a5d9bfa0ec261abe178776c4140fa1e0ffacee57 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Sep 2017 10:34:04 +0000 Subject: [PATCH 398/402] Upgrades windshaft to 3.3.2 --- NEWS.md | 1 + package.json | 2 +- yarn.lock | 359 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 265 insertions(+), 97 deletions(-) diff --git a/NEWS.md b/NEWS.md index 15720ece..0aad89fd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## 3.12.10 Released 2017-mm-dd + - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). ## 3.12.9 Released 2017-09-07 diff --git a/package.json b/package.json index 12430490..7e4f4b52 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.3.1", + "windshaft": "3.3.2", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 8270d25a..a9ba816b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -48,10 +52,6 @@ ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" -ap@~0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" - aproba@^1.0.3: version "1.1.2" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" @@ -205,7 +205,7 @@ camshaft@0.58.1: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -343,12 +343,16 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0, commander@^2.9.0: +commander@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" +commander@^2.9.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -368,8 +372,8 @@ content-disposition@0.5.1: resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.1.tgz#87476c6a67c8daa87e32e87616df883ba7fb071b" content-type@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.2.tgz#b7d113aee7a8dd27bd21133c4dc2529df1721eed" + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" cookie-signature@1.0.6: version "1.0.6" @@ -403,7 +407,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@^2.2.0, debug@~2.2.0: +debug@2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -421,6 +425,12 @@ debug@^1.0.4: dependencies: ms "2.0.0" +debug@^2.2.0: + version "2.6.8" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" + dependencies: + ms "2.0.0" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -431,7 +441,7 @@ deep-eql@^0.1.3: dependencies: type-detect "0.1.1" -deep-equal@^1.0.0: +deep-equal@^1.0.0, deep-equal@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" @@ -443,6 +453,17 @@ deep-is@~0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" +define-properties@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94" + dependencies: + foreach "^2.0.5" + object-keys "^1.0.8" + +defined@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -491,7 +512,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -525,6 +550,24 @@ error-ex@^1.2.0: dependencies: is-arrayish "^0.2.1" +es-abstract@^1.5.0: + version "1.8.2" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.8.2.tgz#25103263dc4decbda60e0c737ca32313518027ee" + dependencies: + es-to-primitive "^1.1.1" + function-bind "^1.1.1" + has "^1.0.1" + is-callable "^1.1.3" + is-regex "^1.0.4" + +es-to-primitive@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.1.1.tgz#45355248a88979034b6792e19bb81f2b7975dd0d" + dependencies: + is-callable "^1.1.1" + is-date-object "^1.0.1" + is-symbol "^1.0.1" + es6-promise@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.1.2.tgz#795e25ceb47f7babb263d151afbedd92d18e6a07" @@ -636,6 +679,16 @@ find-up@^1.0.0: path-exists "^2.0.0" pinkie-promise "^2.0.0" +for-each@~0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.2.tgz#2c40450b9348e97f281322593ba96704b9abd4d4" + dependencies: + is-function "~1.0.0" + +foreach@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -649,8 +702,8 @@ form-data@~2.1.1: mime-types "^2.1.12" forwarded@~0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.0.tgz#19ef9874c4ae1c297bcf078fde63a09b66a84363" + version "0.1.2" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" fresh@0.3.0: version "0.3.0" @@ -677,6 +730,10 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: mkdirp ">=0.5 0" rimraf "2" +function-bind@^1.0.2, function-bind@^1.1.1, function-bind@~1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + gauge@~2.7.3: version "2.7.4" resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" @@ -691,11 +748,11 @@ gauge@~2.7.3: wide-align "^1.1.0" gdal@~0.9.2: - version "0.9.4" - resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.4.tgz#6dad4abb8ffb3e0d51150fb7935ad7c622c81818" + version "0.9.6" + resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.6.tgz#0cf75d830d35847b4274368b10e04a925321a0ba" dependencies: - nan "~2.5.0" - node-pre-gyp "~0.6.27" + nan "~2.6.2" + node-pre-gyp "~0.6.36" generate-function@^2.0.0: version "2.0.0" @@ -733,7 +790,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -764,6 +821,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5, glob@^7.1.1, glob@~7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -834,6 +902,12 @@ has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" +has@^1.0.1, has@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.1.tgz#8461733f538b0837c9361e39a9ab9e9704dc2f28" + dependencies: + function-bind "^1.0.2" + hawk@~3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" @@ -928,15 +1002,27 @@ is-builtin-module@^1.0.0: dependencies: builtin-modules "^1.0.0" +is-callable@^1.1.1, is-callable@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.3.tgz#86eb75392805ddc33af71c92a0eedf74ee7604b2" + +is-date-object@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.1.tgz#9aa20eb6aeebbff77fbd33e74ca01b33581d3a16" + is-fullwidth-code-point@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" dependencies: number-is-nan "^1.0.0" +is-function@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-function/-/is-function-1.0.1.tgz#12cfb98b65b57dd3d193a3121f5f6e2f437602b5" + is-my-json-valid@^2.12.4: - version "2.16.0" - resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.0.tgz#f079dd9bfdae65ee2038aae8acbc86ab109e3693" + version "2.16.1" + resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11" dependencies: generate-function "^2.0.0" generate-object-property "^1.1.0" @@ -947,6 +1033,16 @@ is-property@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84" +is-regex@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491" + dependencies: + has "^1.0.1" + +is-symbol@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.1.tgz#3cc59f00025194b6ab2e38dbae6689256b660572" + is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -991,16 +1087,16 @@ istanbul@~0.4.3: wordwrap "^1.0.0" js-base64@^2.1.9: - version "2.1.9" - resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" + version "2.3.2" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.3.2.tgz#a79a923666372b580f8e27f51845c6f7e8fbfbaf" js-string-escape@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" js-yaml@3.x, js-yaml@^3.4.6: - version "3.9.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.9.1.tgz#08775cebdfdd359209f0d2acd383c8f86a6904a0" + version "3.10.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" dependencies: argparse "^1.0.7" esprima "^4.0.0" @@ -1231,17 +1327,17 @@ millstone@0.6.17: underscore "~1.6.0" zipfile "~0.5.5" -mime-db@~1.29.0: - version "1.29.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.29.0.tgz#48d26d235589651704ac5916ca06001914266878" +mime-db@~1.30.0: + version "1.30.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: - version "2.1.16" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.16.tgz#2b858a52e5ecd516db897ac2be87487830698e23" + version "2.1.17" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: - mime-db "~1.29.0" + mime-db "~1.30.0" -mime@1.3.4, mime@~1.3.4: +mime@1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1249,20 +1345,28 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +mime@~1.3.4: + version "1.3.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" + +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" -minimist@^1.2.0: +minimist@^1.2.0, minimist@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1313,17 +1417,17 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" nan@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" -nan@~2.5.0: - version "2.5.1" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" +nan@~2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" ncp@~2.0.0: version "2.0.0" @@ -1344,9 +1448,9 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36: - version "0.6.36" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" +node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.37: + version "0.6.37" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.37.tgz#3c872b236b2e266e4140578fe1ee88f693323a05" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1355,6 +1459,7 @@ node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36: request "^2.81.0" rimraf "^2.6.1" semver "^5.3.0" + tape "^4.6.3" tar "^2.2.1" tar-pack "^3.4.0" @@ -1409,6 +1514,14 @@ object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" +object-inspect@~1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.3.0.tgz#5b1eb8e6742e2ee83342a637034d844928ba2f6d" + +object-keys@^1.0.8: + version "1.0.11" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1475,8 +1588,8 @@ parse-json@^2.2.0: error-ex "^1.2.0" parseurl@~1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56" + version "1.3.2" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" path-exists@^2.0.0: version "2.1.0" @@ -1488,6 +1601,10 @@ path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" +path-parse@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -1516,16 +1633,15 @@ pg-pool@1.*: object-assign "4.1.0" pg-types@1.*: - version "1.12.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" + version "1.12.1" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2" dependencies: - ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: +"pg@github:cartodb/node-postgres#6.1.6-cdb1": version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1700,7 +1816,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1730,6 +1846,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1747,32 +1872,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.69.0, request@^2.81.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1799,6 +1899,31 @@ request@^2.69.0, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1811,6 +1936,18 @@ resolve@1.1.x: version "1.1.7" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" +resolve@~1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.4.0.tgz#a75be01c53da25d934a98ebd0e4c4a7312f92a86" + dependencies: + path-parse "^1.0.5" + +resumer@~0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/resumer/-/resumer-0.0.0.tgz#f1e8f461e4064ba39e82af3cdc2a8c893d076759" + dependencies: + through "~2.3.4" + right-align@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/right-align/-/right-align-0.1.3.tgz#61339b722fe6a3515689210d24e14c96148613ef" @@ -1818,8 +1955,8 @@ right-align@^0.1.1: align-text "^0.1.1" rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" + version "2.6.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" dependencies: glob "^7.0.5" @@ -1928,8 +2065,8 @@ source-map@^0.4.4: amdefine ">=0.0.4" source-map@^0.5.1, source-map@^0.5.6, source-map@~0.5.1: - version "0.5.6" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412" + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" source-map@~0.2.0: version "0.2.0" @@ -1955,10 +2092,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -1970,11 +2111,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.9" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.9.tgz#b2e7fbaa348380318d3834323918c3c351b8bf18" + version "3.1.11" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.11.tgz#84421b181b3e9f4939b7a3d712263d7946cfc96d" dependencies: - nan "~2.6.2" - node-pre-gyp "~0.6.36" + nan "~2.7.0" + node-pre-gyp "~0.6.37" srs@1.x: version "1.2.0" @@ -2026,6 +2167,14 @@ string-width@^1.0.1, string-width@^1.0.2: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" +string.prototype.trim@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz#d04de2c89e137f4d7d206f086b5ed2fae6be8cea" + dependencies: + define-properties "^1.1.2" + es-abstract "^1.5.0" + function-bind "^1.0.2" + string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" @@ -2076,6 +2225,24 @@ supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: dependencies: has-flag "^1.0.0" +tape@^4.6.3: + version "4.8.0" + resolved "https://registry.yarnpkg.com/tape/-/tape-4.8.0.tgz#f6a9fec41cc50a1de50fa33603ab580991f6068e" + dependencies: + deep-equal "~1.0.1" + defined "~1.0.0" + for-each "~0.3.2" + function-bind "~1.1.0" + glob "~7.1.2" + has "~1.0.1" + inherits "~2.0.3" + minimist "~1.2.0" + object-inspect "~1.3.0" + resolve "~1.4.0" + resumer "~0.0.0" + string.prototype.trim "~1.1.2" + through "~2.3.8" + tar-pack@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.0.tgz#23be2d7f671a8339376cbdb0b8fe3fdebf317984" @@ -2104,11 +2271,11 @@ through2@~0.2.3: readable-stream "~1.1.9" xtend "~2.1.1" -through@2: +through@2, through@~2.3.4, through@~2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2116,7 +2283,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2281,9 +2448,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.1.tgz#b5557fa6b0cfa13920904f57206b86f7aa054f74" +windshaft@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.2.tgz#72efe0dbc0d8d4bcba4211fdabd15dd2e0799df9" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From f133d983e85dfe9831cfb3d7d56653028f8c575f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Sep 2017 12:46:40 +0000 Subject: [PATCH 399/402] Release 3.12.10 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 0aad89fd..037467f8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.12.10 -Released 2017-mm-dd +Released 2017-09-18 - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). ## 3.12.9 From 33121871b0df4d90f8d6712301e6509f57b5618e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 18 Sep 2017 12:47:37 +0000 Subject: [PATCH 400/402] Stubs next version --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 037467f8..2efa11b6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 3.12.11 +Released 2017-mm-dd + ## 3.12.10 Released 2017-09-18 - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). diff --git a/package.json b/package.json index 7e4f4b52..9c70ad72 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.10", + "version": "3.12.11", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 5cc31cabe2166b3a7599fb9eb3326905f94f46a4 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 22 Sep 2017 15:41:58 +0000 Subject: [PATCH 401/402] Fix and enable some old tests related to analyses use cases --- .../analysis/analysis-layers-use-cases.js | 716 ++++++++---------- 1 file changed, 306 insertions(+), 410 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 83c2f535..b08c264a 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client'); var dot = require('dot'); var debug = require('debug')('windshaft:cartodb:test'); -describe('analysis-layers use cases', function() { +describe('analysis-layers use cases', function () { - var multitypeStyleTemplate = dot.template([ - "#points['mapnik::geometry_type'=1] {", - " marker-fill-opacity: {{=it._opacity}};", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: {{=it._opacity}};", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: {{=it._color}};", - " marker-allow-overlap: true;", - "}", - "#lines['mapnik::geometry_type'=2] {", - " line-color: {{=it._color}};", - " line-width: 2;", - " line-opacity: {{=it._opacity}};", - "}", - "#polygons['mapnik::geometry_type'=3] {", - " polygon-fill: {{=it._color}};", - " polygon-opacity: {{=it._opacity}};", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: {{=it._opacity}};", - "}" - ].join('\n')); + var multitypeStyleTemplate = dot.template( + `#points['mapnik::geometry_type'=1] { + marker-fill-opacity: {{=it._opacity}}; + marker-line-color: #FFF; + marker-line-width: 0.5; + marker-line-opacity: {{=it._opacity}}; + marker-placement: point; + marker-type: ellipse; + marker-width: 8; + marker-fill: {{=it._color}}; + marker-allow-overlap: true; + } + #lines['mapnik::geometry_type'=2] { + line-color: {{=it._color}}; + line-width: 2; + line-opacity: {{=it._opacity}}; + } + #polygons['mapnik::geometry_type'=3] { + polygon-fill: {{=it._color}}; + polygon-opacity: {{=it._opacity}}; + line-color: #FFF; + line-width: 0.5; + line-opacity: {{=it._opacity}}; + }` + ); function cartocss(color, opacity) { @@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() { version: '1.5.0', layers: layers, dataviews: dataviews || {}, - analysis: analysis || [] + analyses: analysis || [] }; } - function analysisDef(analysis) { - return JSON.stringify(analysis); - } - var DEFAULT_MULTITYPE_STYLE = cartocss(); var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 }; + var pointInPolygonDef = { + id: 'a1', + type: 'point-in-polygon', + params: { + points_source: { + type: 'source', + params: { + query: 'select * from analysis_rent_listings' + } + }, + polygons_source: { + type: 'buffer', + params: { + source: { + type: 'source', + params: { + query: 'select * from analysis_banks' + } + }, + radius: 250 + } + } + } + }; + + var bufferDef = { + id: 'b1', + type: 'buffer', + params: { + source: { + type: 'source', + params: { + query: 'select * from analysis_banks' + } + }, + radius: 250 + } + }; + var useCases = [ { desc: '1 mapnik layer', @@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } @@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_banks", + sql: 'select * from analysis_banks', cartocss: cartocss('#2167AB'), cartocss_version: '2.3.0' } @@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } @@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } }, { - type: 'analysis', + type: 'cartodb', options: { - def: analysisDef({ - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - }), - cartocss: cartocss('black', 0.5) + source: { + id: 'b1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } - ]) + ], + {}, + [ + bufferDef + ] + ) }, { @@ -137,235 +169,98 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } }, { - type: 'analysis', + type: 'cartodb', options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - } - } - }), - cartocss: cartocss('green', 1.0) + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } - ]) + ], + {}, + [ + pointInPolygonDef + ] + ) }, { desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - }, - { - type: 'cartodb', - options: { - sql: "select * from analysis_rent_listings", - cartocss: DEFAULT_MULTITYPE_STYLE, - cartocss_version: '2.3.0' - } - } - ]) - }, - - { - desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'cartodb', - options: { - sql: "select * from analysis_rent_listings", - cartocss: DEFAULT_MULTITYPE_STYLE, - cartocss_version: '2.3.0' - } - }, - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 300 - } - }), - cartocss: cartocss('magenta', 0.5) - } - }, - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 300 - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - } - ]) - }, - - { - skip: true, - desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'cartodb', - options: { - "source": { id: "a" }, - "cartocss": DEFAULT_MULTITYPE_STYLE, - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "b1" }, - "cartocss": cartocss('green', 1.0), - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "b2" }, - "cartocss": cartocss('magenta', 0.5), - "cartocss_version": "2.3.0" - } - } - ], - [ - { - id: "b2", - options: { - def: analysisDef({ - "type": "count-in-polygon", - "id": "a0", - "params": { - "columnName": 'count_airbnb', - "pointsSource": { - "type": "source", - "params": { - query: "select * from analysis_rent_listings" - }, - dataviews: { - price_histogram: { - type: 'histogram', - options: { - column: 'price' - } - } - } - }, - "polygonsSource": { - "id": "b1", - "type": "buffer", - "params": { - "source": { - "id": "b0", - "type": "source", - "params": { - query: "select * from analysis_banks" - } - }, - "radius": 250 - }, - dataviews: { - bank_category: { - type: 'aggregation', - options: { - column: 'bank' - } - } - } - } + mapConfig: mapConfig( + [ + { + type: 'cartodb', + options: { + source: { + id: 'a1' }, - dataviews: { - count_histogram: { - type: 'histogram', - options: { - column: 'count_airbnb' - } - } - } - }), - cartocss: cartocss('green', 1.0) + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + sql: 'select * from analysis_rent_listings', + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } } - } - ]) + ], + {}, + [ + pointInPolygonDef + ] + ) + }, + + { + desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', + mapConfig: mapConfig( + [ + { + type: 'cartodb', + options: { + sql: 'select * from analysis_rent_listings', + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + source: { + id: 'b1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + } + ], + {}, + [ + bufferDef, + pointInPolygonDef + ] + ) }, { @@ -377,55 +272,55 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - "source": { id: "b0" }, - "cartocss": [ - "#distribution_centers {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 0.7;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: blue;", - " marker-allow-overlap: true;", - "}" + 'source': { id: 'b0' }, + 'cartocss': [ + '#distribution_centers {', + ' marker-fill-opacity: 1.0;', + ' marker-line-color: #FFF;', + ' marker-line-width: 0.5;', + ' marker-line-opacity: 0.7;', + ' marker-placement: point;', + ' marker-type: ellipse;', + ' marker-width: 8;', + ' marker-fill: blue;', + ' marker-allow-overlap: true;', + '}' ].join('\n'), - "cartocss_version": "2.3.0" + 'cartocss_version': '2.3.0' } }, { type: 'cartodb', options: { - "source": { id: "a0" }, - "cartocss": [ - "#shops {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 0.7;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: red;", - " marker-allow-overlap: true;", - "}" + 'source': { id: 'a0' }, + 'cartocss': [ + '#shops {', + ' marker-fill-opacity: 1.0;', + ' marker-line-color: #FFF;', + ' marker-line-width: 0.5;', + ' marker-line-opacity: 0.7;', + ' marker-placement: point;', + ' marker-type: ellipse;', + ' marker-width: 8;', + ' marker-fill: red;', + ' marker-allow-overlap: true;', + '}' ].join('\n'), - "cartocss_version": "2.3.0" + 'cartocss_version': '2.3.0' } }, { type: 'cartodb', options: { - "source": { id: "a1" }, - "cartocss": [ - "#routing {", - " line-color: ramp([routing_time], colorbrewer(Reds));", - " line-width: ramp([routing_time], 2, 8);", - " line-opacity: 1.0;", - "}" + 'source': { id: 'a1' }, + 'cartocss': [ + '#routing {', + ' line-color: ramp([routing_time], colorbrewer(Reds));', + ' line-width: ramp([routing_time], 2, 8);', + ' line-opacity: 1.0;', + '}' ].join('\n'), - "cartocss_version": "2.3.0" + 'cartocss_version': '2.3.0' } } ], @@ -485,91 +380,92 @@ describe('analysis-layers use cases', function() { skip: true, desc: 'II. Population analysis', mapConfig: mapConfig( - // layers - [ + // layers + [ + { + type: 'cartodb', + options: { + 'source': { id: 'a2' }, + 'cartocss': [ + '#count_in_polygon {', + ' polygon-opacity: 1.0', + ' line-color: #FFF;', + ' line-width: 0.5;', + ' line-opacity: 0.7', + ' polygon-fill: ramp([estimated_people], colorbrewer(Reds));', + '}' + ].join('\n'), + 'cartocss_version': '2.3.0' + } + }, + { + type: 'cartodb', + options: { + 'source': { id: 'a0' }, + 'cartocss': DEFAULT_MULTITYPE_STYLE, + 'cartocss_version': '2.3.0' + } + } + ], + // dataviews { - type: 'cartodb', - options: { - "source": { id: "a2" }, - "cartocss": [ - "#count_in_polygon {", - " polygon-opacity: 1.0", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: 0.7", - " polygon-fill: ramp([estimated_people], colorbrewer(Reds));", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" + total_population_formula: { + 'source': { id: 'a3' }, + type: 'formula', + options: { + column: 'total_population', + operation: 'sum' + } + }, + people_histogram: { // this injects a range filter at `a2` node output + 'source': { id: 'a2' }, + type: 'histogram', + options: { + column: 'estimated_people' + } + }, + subway_line_category: { // this injects a category filter at `a0` node output + 'source': { id: 'a0' }, + type: 'aggregation', + options: { + column: 'subway_line' + } } }, - { - type: 'cartodb', - options: { - "source": { id: "a0" }, - "cartocss": DEFAULT_MULTITYPE_STYLE, - "cartocss_version": "2.3.0" - } - } - ], - // dataviews - { - total_population_formula: { - "source": { id: "a3" }, - type: 'formula', - options: { - column: 'total_population', - operation: 'sum' - } - }, - people_histogram: { // this injects a range filter at `a2` node output - "source": { id: "a2" }, - type: 'histogram', - options: { - column: 'estimated_people' - } - }, - subway_line_category: { // this injects a category filter at `a0` node output - "source": { id: "a0" }, - type: 'aggregation', - options: { - column: 'subway_line' - } - } - }, - // analysis - [ - { - id: 'a3', - // this will union the polygons, produce just one polygon, and calculate the total population for it - type: 'total-population', - params: { - columnName: 'total_population', - source: { - id: 'a2', - type: 'estimated-population', - params: { - columnName: 'estimated_people', - source: { - id: 'a1', - type: 'trade-area', - params: { - source: { - "id": "a0", - "type": "source", - "params": { - query: "select * from subway_stops" - } - }, - kind: 'walk', - time: 300 + // analysis + [ + { + id: 'a3', + // this will union the polygons, produce just one polygon, + // and calculate the total population for it + type: 'total-population', + params: { + columnName: 'total_population', + source: { + id: 'a2', + type: 'estimated-population', + params: { + columnName: 'estimated_people', + source: { + id: 'a1', + type: 'trade-area', + params: { + source: { + 'id': 'a0', + 'type': 'source', + 'params': { + query: 'select * from subway_stops' + } + }, + kind: 'walk', + time: 300 + } } } } } } - } - ]) + ]) }, { @@ -581,38 +477,38 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - "source": { id: "a1" }, - "cartocss": [ - "#count_in_polygon {", - " polygon-opacity: 1.0", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: 0.7", - " polygon-fill: ramp([count_people], colorbrewer(Reds));", - "}" + 'source': { id: 'a1' }, + 'cartocss': [ + '#count_in_polygon {', + ' polygon-opacity: 1.0', + ' line-color: #FFF;', + ' line-width: 0.5;', + ' line-opacity: 0.7', + ' polygon-fill: ramp([count_people], colorbrewer(Reds));', + '}' ].join('\n'), - "cartocss_version": "2.3.0" + 'cartocss_version': '2.3.0' } } ], // dataviews { age_histogram: { - "source": { id: "a0" }, + 'source': { id: 'a0' }, type: 'histogram', options: { column: 'age' } }, income_histogram: { - "source": { id: "a0" }, + 'source': { id: 'a0' }, type: 'histogram', options: { column: 'income' } }, gender_category: { - "source": { id: "a0" }, + 'source': { id: 'a0' }, type: 'aggregation', options: { column: 'gender' @@ -622,22 +518,22 @@ describe('analysis-layers use cases', function() { // analysis [ { - "id": "a1", - "type": "count-in-polygon", - "params": { - "columnName": 'count_people', - "pointsSource": { - "id": 'a0', - "type": "source", - "params": { - query: "select the_geom, age, gender, income from people" + 'id': 'a1', + 'type': 'count-in-polygon', + 'params': { + 'columnName': 'count_people', + 'pointsSource': { + 'id': 'a0', + 'type': 'source', + 'params': { + query: 'select the_geom, age, gender, income from people' } }, - "polygonsSource": { - "id": "b0", - "type": "source", - "params": { - query: "select * from postal_codes" + 'polygonsSource': { + 'id': 'b0', + 'type': 'source', + 'params': { + query: 'select * from postal_codes' } } } @@ -648,20 +544,20 @@ describe('analysis-layers use cases', function() { ]; - useCases.forEach(function(useCase, imageIdx) { + useCases.forEach(function (useCase) { if (!!useCase.skip) { - debug(JSON.stringify(useCase.mapConfig, null, 4)); + return debug(JSON.stringify(useCase.mapConfig, null, 4)); } - it.skip('should implement use case: "' + useCase.desc + '"', function(done) { + it(`should implement use case: '${useCase.desc}'`, function (done) { var testClient = new TestClient(useCase.mapConfig, 1234); var tile = useCase.tile || TILE_ANALYSIS_TABLES; - testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { + testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) { assert.ok(!err, err); - image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); + //image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); assert.equal(image.width(), 256); From 3134f40eac32eaa7a302a69d0c1cf81fa9f114db Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 22 Sep 2017 15:42:52 +0000 Subject: [PATCH 402/402] Remove advanced use cases that no longer make sense --- .../analysis/analysis-layers-use-cases.js | 280 ------------------ 1 file changed, 280 deletions(-) diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index b08c264a..2f087626 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -261,287 +261,7 @@ describe('analysis-layers use cases', function () { pointInPolygonDef ] ) - }, - - { - skip: true, - desc: 'I. Distribution centers', - mapConfig: mapConfig( - // layers - [ - { - type: 'cartodb', - options: { - 'source': { id: 'b0' }, - 'cartocss': [ - '#distribution_centers {', - ' marker-fill-opacity: 1.0;', - ' marker-line-color: #FFF;', - ' marker-line-width: 0.5;', - ' marker-line-opacity: 0.7;', - ' marker-placement: point;', - ' marker-type: ellipse;', - ' marker-width: 8;', - ' marker-fill: blue;', - ' marker-allow-overlap: true;', - '}' - ].join('\n'), - 'cartocss_version': '2.3.0' - } - }, - { - type: 'cartodb', - options: { - 'source': { id: 'a0' }, - 'cartocss': [ - '#shops {', - ' marker-fill-opacity: 1.0;', - ' marker-line-color: #FFF;', - ' marker-line-width: 0.5;', - ' marker-line-opacity: 0.7;', - ' marker-placement: point;', - ' marker-type: ellipse;', - ' marker-width: 8;', - ' marker-fill: red;', - ' marker-allow-overlap: true;', - '}' - ].join('\n'), - 'cartocss_version': '2.3.0' - } - }, - { - type: 'cartodb', - options: { - 'source': { id: 'a1' }, - 'cartocss': [ - '#routing {', - ' line-color: ramp([routing_time], colorbrewer(Reds));', - ' line-width: ramp([routing_time], 2, 8);', - ' line-opacity: 1.0;', - '}' - ].join('\n'), - 'cartocss_version': '2.3.0' - } - } - ], - // dataviews - { - distribution_center_name_category: { - source: { id: 'b0' }, - type: 'aggregation', - options: { - column: 'name' - } - }, - time_histogram: { - source: { id: 'a1' }, - type: 'histogram', - options: { - column: 'routing_time' - } - }, - distance_histogram: { - source: { id: 'a1' }, - type: 'histogram', - options: { - column: 'routing_distance' - } - } - }, - // analysis - [ - { - id: 'a1', - type: 'routing-n-to-n', - params: { - // distanceColumn: 'routing_distance', - // timeColumn: 'routing_time', - originSource: { - id: 'b0', - type: 'source', - params: { - query: 'select * from distribution_centers' - } - }, - destinationSource: { - id: 'a0', - type: 'source', - params: { - query: 'select * from shops' - } - } - } - } - ] - ) - }, - - { - skip: true, - desc: 'II. Population analysis', - mapConfig: mapConfig( - // layers - [ - { - type: 'cartodb', - options: { - 'source': { id: 'a2' }, - 'cartocss': [ - '#count_in_polygon {', - ' polygon-opacity: 1.0', - ' line-color: #FFF;', - ' line-width: 0.5;', - ' line-opacity: 0.7', - ' polygon-fill: ramp([estimated_people], colorbrewer(Reds));', - '}' - ].join('\n'), - 'cartocss_version': '2.3.0' - } - }, - { - type: 'cartodb', - options: { - 'source': { id: 'a0' }, - 'cartocss': DEFAULT_MULTITYPE_STYLE, - 'cartocss_version': '2.3.0' - } - } - ], - // dataviews - { - total_population_formula: { - 'source': { id: 'a3' }, - type: 'formula', - options: { - column: 'total_population', - operation: 'sum' - } - }, - people_histogram: { // this injects a range filter at `a2` node output - 'source': { id: 'a2' }, - type: 'histogram', - options: { - column: 'estimated_people' - } - }, - subway_line_category: { // this injects a category filter at `a0` node output - 'source': { id: 'a0' }, - type: 'aggregation', - options: { - column: 'subway_line' - } - } - }, - // analysis - [ - { - id: 'a3', - // this will union the polygons, produce just one polygon, - // and calculate the total population for it - type: 'total-population', - params: { - columnName: 'total_population', - source: { - id: 'a2', - type: 'estimated-population', - params: { - columnName: 'estimated_people', - source: { - id: 'a1', - type: 'trade-area', - params: { - source: { - 'id': 'a0', - 'type': 'source', - 'params': { - query: 'select * from subway_stops' - } - }, - kind: 'walk', - time: 300 - } - } - } - } - } - } - ]) - }, - - { - skip: true, - desc: 'III. Point in polygon', - mapConfig: mapConfig( - // layers - [ - { - type: 'cartodb', - options: { - 'source': { id: 'a1' }, - 'cartocss': [ - '#count_in_polygon {', - ' polygon-opacity: 1.0', - ' line-color: #FFF;', - ' line-width: 0.5;', - ' line-opacity: 0.7', - ' polygon-fill: ramp([count_people], colorbrewer(Reds));', - '}' - ].join('\n'), - 'cartocss_version': '2.3.0' - } - } - ], - // dataviews - { - age_histogram: { - 'source': { id: 'a0' }, - type: 'histogram', - options: { - column: 'age' - } - }, - income_histogram: { - 'source': { id: 'a0' }, - type: 'histogram', - options: { - column: 'income' - } - }, - gender_category: { - 'source': { id: 'a0' }, - type: 'aggregation', - options: { - column: 'gender' - } - } - }, - // analysis - [ - { - 'id': 'a1', - 'type': 'count-in-polygon', - 'params': { - 'columnName': 'count_people', - 'pointsSource': { - 'id': 'a0', - 'type': 'source', - 'params': { - query: 'select the_geom, age, gender, income from people' - } - }, - 'polygonsSource': { - 'id': 'b0', - 'type': 'source', - 'params': { - query: 'select * from postal_codes' - } - } - } - } - ] - ) } - ]; useCases.forEach(function (useCase) {