diff --git a/.gitignore b/.gitignore index b39f2801..ddeef2ef 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,6 @@ tools/munin/windshaft.conf logs/ pids/ redis.pid -test.log -npm-debug.log +*.log coverage/ +.DS_Store diff --git a/.jshintrc b/.jshintrc index 00a59a27..fbc4c96e 100644 --- a/.jshintrc +++ b/.jshintrc @@ -40,7 +40,7 @@ "debug" : false, // true: Allow debugger statements e.g. browser breakpoints. // "eqnull" : false, // true: Tolerate use of `== null` // "es5" : false, // true: Allow ES5 syntax (ex: getters and setters) -// "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`) + "esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`) // "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features) // // (ex: `for each`, multiple try/catch, function expression…) // "evil" : false, // true: Tolerate use of `eval` and `new Function()` diff --git a/HOWTO_RELEASE b/HOWTO_RELEASE index be552a95..a2927b60 100644 --- a/HOWTO_RELEASE +++ b/HOWTO_RELEASE @@ -1,7 +1,7 @@ 1. Test (make clean all check), fix if broken before proceeding 2. Ensure proper version in package.json 3. Ensure NEWS section exists for the new version, review it, add release date -4. Recreate yarn.lock with: `yarn upgrade` +4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}` 5. Commit package.json, yarn.lock, NEWS 6. git tag -a Major.Minor.Patch # use NEWS section as content 7. Stub NEWS/package for next version diff --git a/NEWS.md b/NEWS.md index ade60a55..657f8545 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,284 @@ # Changelog - -## 3.1.2 +## 3.12.11 Released 2017-mm-dd +Bugfixes: + - Bounding box parameter ignored in static named maps #735. + + +## 3.12.10 +Released 2017-09-18 + - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). + +## 3.12.9 +Released 2017-09-07 + +Bug fixes: +- Do not use distinct when calculating quantiles. #743 + +## 3.12.8 +Released 2017-09-07 + +Bug fixes: +- Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962) + +## 3.12.7 +Released 2017-09-01 + + - Upgrades camshaft to [0.58.1](https://github.com/CartoDB/camshaft/releases/tag/0.58.1). + + +## 3.12.6 +Released 2017-08-31 + + - Upgrades camshaft to [0.58.0](https://github.com/CartoDB/camshaft/releases/tag/0.58.0). + + +## 3.12.5 +Released 2017-08-24 + + - Upgrades camshaft to [0.57.0](https://github.com/CartoDB/camshaft/releases/tag/0.57.0). + + +## 3.12.4 +Released 2017-08-23 + +Announcements: + - Upgrades camshaft to [0.56.0](https://github.com/CartoDB/camshaft/releases/tag/0.56.0). + +## 3.12.3 +Released 2017-08-22 + +Announcements: + - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8). + +## 3.12.2 +Released 2017-08-16 + +Bug fixes: + - Polygon count problems #725. + + +## 3.12.1 +Released 2017-08-13 + - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1). + - Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1). + - Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7). + + +## 3.12.0 +Released 2017-08-10 + +Announcements: + - Apply max tile response time for requests to layergoup, tiles, static maps, attributes and dataviews services #717. + - Upgrades windshaft to [3.3.0](https://github.com/CartoDB/windshaft/releases/tag/3.3.0). + - Upgrades cartodb-redis to [0.14.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/0.14.0). + + +## 3.11.0 +Released 2017-08-08 + +Announcements: + - Allow to override with any aggregation for histograms instantiated w/o aggregation. + +Bug fixes: + - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + - Support timestamp with timezones to calculate the number of bins in time-series. + - Fixed issue related to name collision while building time-series query. + + +## 3.10.1 +Released 2017-08-04 + +Bug fixes: + - Exclude Infinities & NaNs from ramps #719. + - Fixed issue in time-series when aggregation starts at 1970-01-01 (epoch) #720. + + +## 3.10.0 +Released 2017-08-03 + +Announcements: + - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698. + - Support special numeric values (±Infinity, NaN) for json responses #706 + + +## 3.9.8 +Released 2017-07-21 + + - Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2). + + +## 3.9.7 +Released 2017-07-20 + +Bug fixes: + - Respond with 204 (No content) when vector tile has no data #712 + +Announcements: + - Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2) + + +## 3.9.6 +Released 2017-07-11 + + - Dataviews: support for aggregation in search results #708 + + +## 3.9.5 +Released 2017-06-27 + + - Dataviews: support special numeric values (±Infinity, NaN) #700 + + +## 3.9.4 +Released 2017-06-22 + +Announcements: + - Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6). + +## 3.9.3 +Released 2017-06-16 + +Announcements: + - Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5). + +## 3.9.2 +Released 2017-06-16 + +Announcements: + - Upgrades camshaft to [0.55.4](https://github.com/CartoDB/camshaft/releases/tag/0.55.4). + +## 3.9.1 +Released 2017-06-06 + +Announcements: + - Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3). + + +## 3.9.0 +Released 2017-05-31 + +Announcements: + - Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1). + - Add support to retrieve info about layer stats in map instantiation. + - Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2). + - Remove promise polyfill from turbo-carto adapter + + +## 3.8.0 +Released 2017-05-22 + +Announcements: + - Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0). + - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1) + + +## 3.7.1 +Released 2017-05-18 + +Bug fixes: + - Fix buffersize assignment when is not defined in requested mapconfig. + + +## 3.7.0 +Released 2017-05-18 + +Announcements: +- Manage multiple values of buffer-size for different formats +- Upgrades windshaft to [3.2.0](https://github.com/CartoDB/windshaft/releases/tag/3.2.0). + + +## 3.6.6 +Released 2017-05-11 + +Announcements: + - Upgrades camshaft to [0.54.4](https://github.com/CartoDB/camshaft/releases/tag/0.54.4). + + +## 3.6.5 +Released 2017-05-09 + +Announcements: + - Upgrades camshaft to [0.54.3](https://github.com/CartoDB/camshaft/releases/tag/0.54.3). + + +## 3.6.4 +Released 2017-05-05 + +Announcements: + - Upgrade cartodb-psql to [0.8.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.8.0). + - Upgrades camshaft to [0.54.2](https://github.com/CartoDB/camshaft/releases/tag/0.54.2). + - Upgrades windshaft to [3.1.2](https://github.com/CartoDB/windshaft/releases/tag/3.1.2). + + +## 3.6.3 +Released 2017-04-25 + +Announcements: + - Upgrades windshaft to [3.1.1](https://github.com/CartoDB/windshaft/releases/tag/3.1.1). + + +## 3.6.2 +Released 2017-04-24 + +Announcements: + - Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3). + + +## 3.6.1 +Released 2017-04-24 + +Announcements: + - Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1). + + +## 3.6.0 +Released 2017-04-20 + +Announcements: + - Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0). + + +## 3.5.1 +Released 2017-04-11 + +Announcements: + - Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1). + + +## 3.5.0 +Released 2017-04-10 + +Bug fixes: + - Fix invalidation of cache for maps with analyses #638. + +Announcements: + - Upgrades camshaft to [0.53.0](https://github.com/CartoDB/camshaft/releases/tag/0.53.0). + + +## 3.4.0 +Released 2017-04-03 + +Announcements: + - Upgrades camshaft to [0.51.0](https://github.com/CartoDB/camshaft/releases/tag/0.51.0). + + +## 3.3.0 +Released 2017-04-03 + +New features: + - Static map endpoints allow specifying the layers to render #653. + + +## 3.2.0 +Released 2017-03-30 + +Announcements: + - Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0). + - Active GC interval. + ## 3.1.1 Released 2017-03-23 diff --git a/app.js b/app.js index 759b87cc..6eb1ef3a 100755 --- a/app.js +++ b/app.js @@ -144,3 +144,17 @@ process.on('SIGHUP', function() { process.on('uncaughtException', function(err) { global.logger.error('Uncaught exception: ' + err.stack); }); + +if (global.gc) { + var gcInterval = Number.isFinite(global.environment.gc_interval) ? + global.environment.gc_interval : + 10000; + + if (gcInterval > 0) { + setInterval(function gcForcedCycle() { + var start = Date.now(); + global.gc(); + global.statsClient.timing('windshaft.gc', Date.now() - start); + }, gcInterval); + } +} diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 6800d4c6..48d03592 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.localhost' @@ -321,8 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true - + layerStats: true } }; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 41e2353d..ba7fec6f 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' @@ -321,7 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: false + layerStats: false } }; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 170e5e5f..1792779c 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' @@ -321,7 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true + layerStats: true } }; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 3c940ac6..374467c0 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -6,6 +6,9 @@ var config = { // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128). // See http://docs.libuv.org/en/latest/threadpool.html ,uv_threadpool_size: undefined + // Time in milliseconds to force GC cycle. + // Disable by using <=0 value. + ,gc_interval: 10000 // Regular expression pattern to extract username // from hostname. Must have a single grabbing block. ,user_from_host: '(.*)' @@ -315,7 +318,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true + layerStats: true } }; diff --git a/docs/static_maps_api.md b/docs/static_maps_api.md index 736d90e0..91db2a6d 100644 --- a/docs/static_maps_api.md +++ b/docs/static_maps_api.md @@ -150,6 +150,10 @@ It is important to note that generated images are cached from the live data refe * Image resolution is set to 72 DPI * JPEG quality is 85% * Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries. +* If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code: + +{% highlight javascript %}attribution: '© OpenStreetMap contributors, © CARTO +{% endhighlight %} ## Examples diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js index 68533fe9..484d66b1 100644 --- a/lib/cartodb/api/auth_api.js +++ b/lib/cartodb/api/auth_api.js @@ -95,9 +95,7 @@ AuthApi.prototype.authorize = function(req, callback) { self.authorizedByAPIKey(user, req, this); }, function checkApiKey(err, authorized){ - if (req.profiler) { - req.profiler.done('authorizedByAPIKey'); - } + req.profiler.done('authorizedByAPIKey'); assert.ifError(err); // if not authorized by api_key, continue @@ -131,9 +129,7 @@ AuthApi.prototype.authorize = function(req, callback) { } self.pgConnection.setDBAuth(user, req.params, function(err) { - if (req.profiler) { - req.profiler.done('setDBAuth'); - } + req.profiler.done('setDBAuth'); callback(err, true); // authorized (or error) }); } diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index ae9cd46b..0f4f0f4c 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -1,3 +1,5 @@ +var step = require('step'); + /** * * @param metadataBackend @@ -13,16 +15,65 @@ function UserLimitsApi(metadataBackend, options) { module.exports = UserLimitsApi; -UserLimitsApi.prototype.getRenderLimits = function (username, callback) { +UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) { var self = this; - this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) { + + var limits = { + cacheOnTimeout: self.options.limits.cacheOnTimeout || false, + render: self.options.limits.render || 0 + }; + + self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { if (err) { return callback(err); } - return callback(null, { - cacheOnTimeout: self.options.limits.cacheOnTimeout || false, - render: renderLimit || self.options.limits.render || 0 - }); + if (timeoutRenderLimit && timeoutRenderLimit.render) { + if (Number.isFinite(timeoutRenderLimit.render)) { + limits.render = timeoutRenderLimit.render; + } + } + + return callback(null, limits); }); }; + +UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { + var self = this; + + step( + function isAuthorized() { + var next = this; + + if (!apiKey) { + return next(null, false); + } + + self.metadataBackend.getUserMapKey(username, function (err, userApiKey) { + if (err) { + return next(err); + } + + return next(null, userApiKey === apiKey); + }); + }, + function getUserTimeoutRenderLimits(err, authorized) { + var next = this; + + if (err) { + return next(err); + } + + self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { + if (err) { + return next(err); + } + + next(null, { + render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic + }); + }); + }, + callback + ); +}; diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index e0515efe..29dcd903 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -43,53 +43,19 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ownFilter = !!ownFilter; var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; - var sourceId = dataviewDefinition.source.id; // node.id - var layer = _.find(mapConfig.obj().layers, function(l) { - return l.options.source && (l.options.source.id === sourceId); - }); - var queryRewriteData = layer && layer.options.query_rewrite_data; - if (queryRewriteData && dataviewDefinition.node.type === 'source') { - queryRewriteData = _.extend({}, queryRewriteData, { - filters: dataviewDefinition.node.filters, - unfiltered_query: dataviewDefinition.sql.own_filter_on - }); - } - if (params.bbox) { var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); query = bboxFilter.sql(query); - if ( queryRewriteData ) { - var bbox_filter_definition = { - type: 'bbox', - options: { - column: 'the_geom_webmercator', - srid: 3857 - }, - params: { - bbox: params.bbox - } - }; - queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition }); - } } + var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); + var dataviewFactory = DataviewFactoryWithOverviews.getFactory( overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } ); - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), - function castNumbers(overrides, val, k) { - if (!Number.isFinite(+val)) { - throw new Error('Invalid number format for parameter \'' + k + '\''); - } - overrides[k] = +val; - return overrides; - }, - {ownFilter: ownFilter} - ); - var dataview = dataviewFactory.getDataview(query, dataviewDefinition); - dataview.getResult(pg, overrideParams, this); + dataview.getResult(pg, getOverrideParams(params, ownFilter), this); }, function returnCallback(err, result) { return callback(err, result); @@ -97,6 +63,56 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ); }; +function getQueryRewriteData(mapConfig, dataviewDefinition, params) { + var sourceId = dataviewDefinition.source.id; // node.id + var layer = _.find(mapConfig.obj().layers, function(l) { + return l.options.source && (l.options.source.id === sourceId); + }); + var queryRewriteData = layer && layer.options.query_rewrite_data; + if (queryRewriteData && dataviewDefinition.node.type === 'source') { + queryRewriteData = _.extend({}, queryRewriteData, { + filters: dataviewDefinition.node.filters, + unfiltered_query: dataviewDefinition.sql.own_filter_on + }); + } + + if (params.bbox && queryRewriteData) { + var bbox_filter_definition = { + type: 'bbox', + options: { + column: 'the_geom_webmercator', + srid: 3857 + }, + params: { + bbox: params.bbox + } + }; + queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition }); + } + + return queryRewriteData; +} + +function getOverrideParams(params, ownFilter) { + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'), + function castNumbers(overrides, val, k) { + if (!Number.isFinite(+val)) { + throw new Error('Invalid number format for parameter \'' + k + '\''); + } + overrides[k] = +val; + return overrides; + }, + {ownFilter: ownFilter} + ); + + // validation will be delegated to the proper dataview + if (params.aggregation !== undefined) { + overrideParams.aggregation = params.aggregation; + } + + return overrideParams; +} + DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { var dataviewName = params.dataviewName; diff --git a/lib/cartodb/backends/layer-stats/empty-layer-stats.js b/lib/cartodb/backends/layer-stats/empty-layer-stats.js new file mode 100644 index 00000000..0760c0b6 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/empty-layer-stats.js @@ -0,0 +1,16 @@ +function EmptyLayerStats(types) { + this._types = types || {}; +} + +EmptyLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +EmptyLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + setImmediate(function() { + callback(null, {}); + }); +}; + +module.exports = EmptyLayerStats; diff --git a/lib/cartodb/backends/layer-stats/factory.js b/lib/cartodb/backends/layer-stats/factory.js new file mode 100644 index 00000000..8aacdb5a --- /dev/null +++ b/lib/cartodb/backends/layer-stats/factory.js @@ -0,0 +1,23 @@ +var LayerStats = require('./layer-stats'); +var EmptyLayerStats = require('./empty-layer-stats'); +var MapnikLayerStats = require('./mapnik-layer-stats'); +var TorqueLayerStats = require('./torque-layer-stats'); + +module.exports = function LayerStatsFactory(type) { + var layerStatsIterator = []; + var selectedType = type || 'ALL'; + + if (selectedType === 'ALL') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true })); + layerStatsIterator.push(new MapnikLayerStats()); + layerStatsIterator.push(new TorqueLayerStats()); + } else if (selectedType === 'mapnik') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true })); + layerStatsIterator.push(new MapnikLayerStats()); + } else if (selectedType === 'torque') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true })); + layerStatsIterator.push(new TorqueLayerStats()); + } + + return new LayerStats(layerStatsIterator); +}; diff --git a/lib/cartodb/backends/layer-stats/layer-stats.js b/lib/cartodb/backends/layer-stats/layer-stats.js new file mode 100644 index 00000000..2464fb22 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/layer-stats.js @@ -0,0 +1,45 @@ +var queue = require('queue-async'); + +function LayerStats(layerStatsIterator) { + this.layerStatsIterator = layerStatsIterator; +} + +LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) { + var self = this; + var stats = []; + + if (!mapConfig.getLayers().length) { + return callback(null, stats); + } + var metaQueue = queue(mapConfig.getLayers().length); + mapConfig.getLayers().forEach(function (layer, layerId) { + var layerType = mapConfig.layerType(layerId); + + for (var i = 0; i < self.layerStatsIterator.length; i++) { + if (self.layerStatsIterator[i].is(layerType)) { + var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]); + metaQueue.defer(getStats, layer, dbConnection); + break; + } + } + }); + + metaQueue.awaitAll(function (err, results) { + if (err) { + return callback(err); + } + + if (!results) { + return callback(null, null); + } + + mapConfig.getLayers().forEach(function (layer, layerIndex) { + stats[layerIndex] = results[layerIndex]; + }); + + return callback(err, stats); + }); + +}; + +module.exports = LayerStats; diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..c060f964 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,28 @@ +var queryUtils = require('../../utils/query-utils'); + +function MapnikLayerStats () { + this._types = { + mapnik: true, + cartodb: true + }; +} + +MapnikLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +MapnikLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); + // This query would gather stats for postgresql table if not exists + dbConnection.query(queryRowCountSql, function (err, res) { + if (err) { + return callback(null, {estimatedFeatureCount: -1}); + } else { + // We decided that the relation is 1 row == 1 feature + return callback(null, {estimatedFeatureCount: res.rows[0].rows}); + } + }); +}; + +module.exports = MapnikLayerStats; diff --git a/lib/cartodb/backends/layer-stats/torque-layer-stats.js b/lib/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..00b4def2 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,16 @@ +function TorqueLayerStats() { + this._types = { + torque: true + }; +} + +TorqueLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +TorqueLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + return callback(null, {}); +}; + +module.exports = TorqueLayerStats; diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js new file mode 100644 index 00000000..b0385bac --- /dev/null +++ b/lib/cartodb/backends/stats.js @@ -0,0 +1,16 @@ +var layerStats = require('./layer-stats/factory'); + +function StatsBackend() { +} + +module.exports = StatsBackend; + +StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { + var enabledFeatures = global.environment.enabledFeatures; + var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; + if (layerStatsEnabled) { + layerStats().getStats(mapConfig, dbConnection, callback); + } else { + return callback(null, []); + } +}; diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index bb728181..75a482fe 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -296,7 +296,7 @@ TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) { // @param callback function(err) // TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) { - + var self = this; template = templateDefaults(template); @@ -430,13 +430,17 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/, _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/; function _replaceVars (str, params) { - //return _.template(str, params); // lazy way, possibly dangerous - // Construct regular expressions for each param + // Construct regular expressions for each param Object.keys(params).forEach(function(k) { str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]); }); return str; } + +function isObject(val) { + return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val)); +} + TemplateMaps.prototype.instance = function(template, params) { var all_params = {}; var phold = template.placeholders || {}; @@ -474,6 +478,13 @@ TemplateMaps.prototype.instance = function(template, params) { // NOTE: we're deep-cloning the layergroup here var layergroup = JSON.parse(JSON.stringify(template.layergroup)); + + if (layergroup.buffersize && isObject(layergroup.buffersize)) { + Object.keys(layergroup.buffersize).forEach(function(k) { + layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10); + }); + } + for (var i=0; i 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + return next(); + }); + }, function finish(err) { done(err); } diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 4d22aeea..b6aa8bdf 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -8,6 +8,7 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); +var allowQueryParams = require('../middleware/allow-query-params'); function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { @@ -32,6 +33,7 @@ NamedMapsController.prototype.register = function(app) { app.get(app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, + allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), this.staticMap.bind(this)); }; @@ -100,9 +102,7 @@ NamedMapsController.prototype.tile = function(req, res) { self.tileBackend.getTile(namedMapProvider, req.params, this); }, function handleImage(err, tile, headers, stats) { - if (req.profiler) { - req.profiler.add(stats); - } + req.profiler.add(stats); if (err) { self.sendError(req, res, err, 'NAMED_MAP_TILE'); } else { @@ -176,10 +176,8 @@ NamedMapsController.prototype.staticMap = function(req, res) { }); }, function handleImage(err, image, headers, stats) { - if (req.profiler) { - req.profiler.done('render-' + format); - req.profiler.add(stats || {}); - } + req.profiler.done('render-' + format); + req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'STATIC_VIZ_MAP'); diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index 3d201049..08ad5322 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -91,9 +91,7 @@ NamedMapsAdminController.prototype.update = function(req, res) { NamedMapsAdminController.prototype.retrieve = function(req, res) { var self = this; - if (req.profiler) { - req.profiler.start('windshaft-cartodb.get_template'); - } + req.profiler.start('windshaft-cartodb.get_template'); var cdbuser = req.context.user; var tpl_id; @@ -127,9 +125,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) { NamedMapsAdminController.prototype.destroy = function(req, res) { var self = this; - if (req.profiler) { - req.profiler.start('windshaft-cartodb.delete_template'); - } + req.profiler.start('windshaft-cartodb.delete_template'); var cdbuser = req.context.user; var tpl_id; @@ -154,9 +150,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) { NamedMapsAdminController.prototype.list = function(req, res) { var self = this; - if ( req.profiler ) { - req.profiler.start('windshaft-cartodb.get_template_list'); - } + req.profiler.start('windshaft-cartodb.get_template_list'); var cdbuser = req.context.user; diff --git a/lib/cartodb/middleware/allow-query-params.js b/lib/cartodb/middleware/allow-query-params.js new file mode 100644 index 00000000..04a27033 --- /dev/null +++ b/lib/cartodb/middleware/allow-query-params.js @@ -0,0 +1,9 @@ +module.exports = function allowQueryParams(params) { + if (!Array.isArray(params)) { + throw new Error('allowQueryParams must receive an Array of params'); + } + return function allowQueryParamsMiddleware(req, res, next) { + req.context.allowedQueryParams = params; + next(); + }; +}; diff --git a/lib/cartodb/middleware/lzma.js b/lib/cartodb/middleware/lzma.js new file mode 100644 index 00000000..d58f16cc --- /dev/null +++ b/lib/cartodb/middleware/lzma.js @@ -0,0 +1,30 @@ +'use strict'; + +var LZMA = require('lzma').LZMA; + +var lzmaWorker = new LZMA(); + +module.exports = function lzmaMiddleware(req, res, next) { + if (!req.query.hasOwnProperty('lzma')) { + return next(); + } + + // Decode (from base64) + var lzma = new Buffer(req.query.lzma, 'base64') + .toString('binary') + .split('') + .map(function(c) { + return c.charCodeAt(0) - 128; + }); + + // Decompress + lzmaWorker.decompress(lzma, function(result) { + try { + delete req.query.lzma; + Object.assign(req.query, JSON.parse(result)); + next(); + } catch (err) { + next(new Error('Error parsing lzma as JSON: ' + err)); + } + }); +}; diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index c15f0506..77f37723 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -5,11 +5,32 @@ var debug = require('debug')('windshaft:widget:aggregation'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' count(1) AS count,', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -18,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', @@ -44,22 +65,25 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', - ' count, categories_count', + 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', + ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); @@ -84,7 +108,7 @@ var TYPE = 'aggregation'; } } */ -function Aggregation(query, options) { +function Aggregation(query, options, queries) { if (!_.isString(options.column)) { throw new Error('Aggregation expects `column` in widget options'); } @@ -108,9 +132,11 @@ function Aggregation(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = new BaseWidget(); @@ -119,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var _query = this.query; var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -140,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } else { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', @@ -155,30 +208,38 @@ Aggregation.prototype.sql = function(psql, override, callback) { return callback(null, aggregationSql); }; -Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { +Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { return [ - "WITH", - [ - summaryQueryTpl({ - _query: query, - _column: column - }), - rankedCategoriesQueryTpl({ - _query: query, - _column: column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: query, - _column: column - }), - categoriesSummaryCountQueryTpl({ - _query: query, - _column: column - }) - ].join(',\n') - ].join('\n'); + "WITH", + [ + filteredQueryTpl({ + _isFloatColumn: isFloatColumn, + _query: this.query, + _column: this.column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + summaryQueryTpl({ + _isFloatColumn: isFloatColumn, + _query: query, + _column: column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + rankedCategoriesQueryTpl({ + _query: query, + _column: column, + _aggregation: this.getAggregationSql(), + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + categoriesSummaryMinMaxQueryTpl({ + _query: query, + _column: column + }), + categoriesSummaryCountQueryTpl({ + _query: query, + _column: column + }) + ].join(',\n') + ].join('\n'); }; var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); @@ -193,6 +254,8 @@ Aggregation.prototype.format = function(result) { var categories = []; var count = 0; var nulls = 0; + var nans = 0; + var infinities = 0; var minValue = 0; var maxValue = 0; var categoriesCount = 0; @@ -202,12 +265,15 @@ Aggregation.prototype.format = function(result) { var firstRow = result.rows[0]; count = firstRow.count; nulls = firstRow.nulls_count; + nans = firstRow.nans_count; + infinities = firstRow.infinities_count; minValue = firstRow.min_val; maxValue = firstRow.max_val; categoriesCount = firstRow.categories_count; result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', + 'max_val', 'categories_count', 'nans_count', 'infinities_count')); }); } @@ -215,6 +281,8 @@ Aggregation.prototype.format = function(result) { aggregation: this.aggregation, count: count, nulls: nulls, + nans: nans, + infinities: infinities, min: minValue, max: maxValue, categoriesCount: categoriesCount, @@ -253,6 +321,8 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { var self = this; var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); + var _value = this.aggregation !== 'count' && this.aggregationColumn ? + this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ @@ -265,7 +335,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { _searchFiltered: filterCategoriesQueryTpl({ _query: this.query, _column: this.column, - _value: 'count(1)', + _value: _value, _userQuery: _userQuery }) }); diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index b2e2f188..29069d37 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -1,3 +1,6 @@ +var dot = require('dot'); +dot.templateSettings.strip = false; + function BaseDataview() {} module.exports = BaseDataview; @@ -5,8 +8,11 @@ module.exports = BaseDataview; BaseDataview.prototype.getResult = function(psql, override, callback) { var self = this; this.sql(psql, override, function(err, query) { - psql.query(query, function(err, result) { + if (err) { + return callback(err); + } + psql.query(query, function(err, result) { if (err) { return callback(err, result); } @@ -24,3 +30,42 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { BaseDataview.prototype.search = function(psql, userQuery, callback) { return callback(null, this.format({ rows: [] })); }; + +var FLOAT_OIDS = { + 700: true, + 701: true, + 1700: true +}; + +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1' +); + +BaseDataview.prototype.getColumnType = function (psql, column, query, callback) { + var readOnlyTransaction = true; + + var columnTypeQuery = columnTypeQueryTpl({ + column: column, query: query + }); + + psql.query(columnTypeQuery, function(err, result) { + if (err) { + return callback(err); + } + var pgType = result.rows[0].pg_typeof; + callback(null, getPGTypeName(pgType)); + }, readOnlyTransaction); +}; + +function getPGTypeName (pgType) { + return { + float: FLOAT_OIDS.hasOwnProperty(pgType), + date: DATE_OIDS.hasOwnProperty(pgType) + }; +} diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 156985c5..7ec356b7 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -7,9 +7,19 @@ dot.templateSettings.strip = false; var formulaQueryTpl = dot.template([ 'SELECT', - '{{=it._operation}}({{=it._column}}) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + ' {{=it._operation}}({{=it._column}}) AS result,', + ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')); var VALID_OPERATIONS = { @@ -31,7 +41,7 @@ var TYPE = 'formula'; } } */ -function Formula(query, options) { +function Formula(query, options, queries) { if (!_.isString(options.operation)) { throw new Error('Formula expects `operation` in widget options'); } @@ -47,8 +57,10 @@ function Formula(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = new BaseWidget(); @@ -57,14 +69,27 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var _query = this.query; + if (this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var formulaSql = formulaQueryTpl({ - _query: _query, + _isFloatColumn: this._isFloatColumn, + _query: this.query, _operation: this.operation, _column: this.column }); @@ -78,13 +103,17 @@ Formula.prototype.format = function(result) { var formattedResult = { operation: this.operation, result: 0, - nulls: 0 + nulls: 0, + nans: 0, + infinities: 0 }; if (result.rows.length) { formattedResult.operation = this.operation; formattedResult.result = result.rows[0].result; formattedResult.nulls = result.rows[0].nulls_count; + formattedResult.nans = result.rows[0].nans_count; + formattedResult.infinities = result.rows[0].infinities_count; } return formattedResult; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5d102bf5..6e62ad9d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -5,108 +5,289 @@ var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var dateIntervalQueryTpl = dot.template([ + 'WITH', + '__cdb_dates AS (', + ' SELECT', + ' MAX({{=it.column}}::timestamp) AS __cdb_end,', + ' MIN({{=it.column}}::timestamp) AS __cdb_start', + ' FROM ({{=it.query}}) __cdb_source', + '),', + '__cdb_interval_in_days AS (', + ' SELECT' , + ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', + ' FROM __cdb_dates', + '),', + '__cdb_interval_in_hours AS (', + ' SELECT', + ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', + ' FROM __cdb_interval_in_days, __cdb_dates', + '),', + '__cdb_interval_in_minutes AS (', + ' SELECT', + ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', + ' FROM __cdb_interval_in_hours, __cdb_dates', + '),', + '__cdb_interval_in_seconds AS (', + ' SELECT', + ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', + ' FROM __cdb_interval_in_minutes, __cdb_dates', + ')', + 'SELECT', + ' ROUND(__cdb_days / 365) AS year,', + ' ROUND(__cdb_days / 90) AS quarter,', + ' ROUND(__cdb_days / 30) AS month,', + ' ROUND(__cdb_days / 7) AS week,', + ' __cdb_days AS day,', + ' __cdb_hours AS hour,', + ' __cdb_minutes AS minute,', + ' __cdb_seconds AS second', + 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' +].join('\n')); + +var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + '__cdb_filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) __cdb_filtered_source_query', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' {{?it._isFloatColumn}}AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', - ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', + ' FROM __cdb_filtered_source', ')' ].join(' \n')); var overrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', + ' FROM __cdb_filtered_source', ')' ].join('\n')); var iqrQueryTpl = dot.template([ - 'iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + '__cdb_iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', ' FROM (', ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', - ' ) _cdb_iqr', + ' ) __cdb_iqr', ')' ].join('\n')); var binsQueryTpl = dot.template([ - 'bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + '__cdb_bins AS (', + ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', - ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' END AS __cdb_bins_number', + ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); var overrideBinsQueryTpl = dot.template([ - 'bins AS (', - ' SELECT {{=it._bins}} AS bins_number', + '__cdb_bins AS (', + ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); var nullsQueryTpl = dot.template([ - 'nulls AS (', + '__cdb_nulls AS (', ' SELECT', - ' count(*) AS nulls_count', - ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' count(*) AS __cdb_nulls_count', + ' FROM ({{=it._query}}) __cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + '__cdb_infinities AS (', + ' SELECT', + ' count(*) AS __cdb_infinities_count', + ' FROM ({{=it._query}}) __cdb_infinities_query', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + '__cdb_nans AS (', + ' SELECT', + ' count(*) AS __cdb_nans_count', + ' FROM ({{=it._query}}) __cdb_nans_query', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', - ' avg_val,', - ' CASE WHEN min_val = max_val', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', + ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', + ' __cdb_nans_count AS nans_count,{{?}}', + ' __cdb_avg_val AS avg_val,', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' ELSE GREATEST(', + ' 1,', + ' LEAST(', + ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', + ' __cdb_bins_number', + ' )', + ' ) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', + ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', + 'GROUP BY bin, bins_number, bin_width, nulls_count,', + ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); +var dateBasicsQueryTpl = dot.template([ + '__cdb_basics AS (', + ' SELECT', + ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', + ' min(date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' )) AS __cdb_start_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', + ' FROM ({{=it._query}}) __cdb_basics_query', + ')' +].join(' \n')); + +var dateOverrideBasicsQueryTpl = dot.template([ + '__cdb_basics AS (', + ' SELECT', + ' max({{=it._end}})::float AS __cdb_max_val,', + ' min({{=it._start}})::float AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', + ' min(', + ' date_trunc(', + ' \'{{=it._aggregation}}\',', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' )', + ' ) AS __cdb_start_date,', + ' max(', + ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' ) AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', + ' FROM ({{=it._query}}) __cdb_basics_query', + ')' +].join(' \n')); + +var dateBinsQueryTpl = dot.template([ + '__cdb_bins AS (', + ' SELECT', + ' __cdb_bins_array,', + ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', + ' FROM (', + ' SELECT', + ' ARRAY(', + ' SELECT GENERATE_SERIES(', + ' __cdb_start_date::timestamptz,', + ' __cdb_end_date::timestamptz,', + ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', + ' )', + ' ) AS __cdb_bins_array', + ' FROM __cdb_basics', + ' ) __cdb_bins_array_query', + ')' +].join('\n')); + +var dateHistogramQueryTpl = dot.template([ + 'SELECT', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', + ' CASE WHEN __cdb_min_val = __cdb_max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(', + ' WIDTH_BUCKET(', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', + ' __cdb_bins_array', + ' ),', + ' __cdb_bins_number', + ' )) - 1', + ' END AS bin,', + ' min(', + ' date_part(', + ' \'epoch\', ', + ' date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' ) AT TIME ZONE \'{{=it._offset}}\'', + ' )', + ' )::numeric AS timestamp,', + ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', + ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', + 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', + 'ORDER BY bin' +].join('\n')); var TYPE = 'histogram'; /** - { - type: 'histogram', - options: { - column: 'name', - bins: 10 // OPTIONAL - } +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} + +Time series: +{ + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) + offset: -7200 // OPTIONAL (UTC offset in seconds) + } } */ function Histogram(query, options, queries) { @@ -118,6 +299,8 @@ function Histogram(query, options, queries) { this.queries = queries; this.column = options.column; this.bins = options.bins; + this.aggregation = options.aggregation; + this.offset = options.offset; this._columnType = null; } @@ -127,50 +310,55 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.queries.no_filters - }); - if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction return null; } + this._buildQuery(psql, override, callback); +}; + +Histogram.prototype.isDateHistogram = function (override) { + return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); +}; + +Histogram.prototype._buildQuery = function (psql, override, callback) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; + var _query = this.query; + + if (this.isDateHistogram(override)) { + return this._buildDateHistogramQuery(psql, override, callback); + } + if (this._columnType === 'date') { _column = columnCastTpl({column: _column}); } - var _query = this.query; + filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', + _query: _query, + _column: _column + }); - var basicsQuery, binsQuery; - - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, @@ -190,7 +378,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -211,18 +399,34 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - basicsQuery, - binsQuery, - nullsQueryTpl({ + if (this._columnType === 'float') { + cteSql.push( + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) @@ -233,6 +437,143 @@ Histogram.prototype.sql = function(psql, override, callback) { return callback(null, histogramSql); }; +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); +}; + +var DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + +Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { + var _column = this.column; + var _query = this.query; + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + + if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + + if (_aggregation === 'auto') { + this.getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + override.aggregation = this.aggregation; + this._buildDateHistogramQuery(psql, override, callback); + }.bind(this)); + return null; + } + + var dateBasicsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end')) { + dateBasicsQuery = dateOverrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _start: getBinStart(override), + _end: getBinEnd(override), + _offset: parseOffset(_offset, _aggregation) + }); + } else { + dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); + } + + var dateBinsQuery = [ + dateBinsQueryTpl({ + _aggregation: _aggregation + }) + ].join(',\n'); + + var nullsQuery = nullsQueryTpl({ + _query: _query, + _column: _column + }); + + var dateHistogramQuery = dateHistogramQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); + + var histogramSql = [ + "WITH", + [ + dateBasicsQuery, + dateBinsQuery, + nullsQuery + ].join(',\n'), + dateHistogramQuery + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + +Histogram.prototype.getAutomaticAggregation = function (psql, callback) { + var dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + debug(dateIntervalQuery); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + var aggegations = result.rows[0]; + var aggregation = Object.keys(aggegations) + .map(function (key) { + return { + name: key, + value: aggegations[key] + }; + }) + .reduce(function (closer, current) { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + var closerDiff = MAX_INTERVAL_VALUE - closer.value; + var currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; @@ -241,7 +582,12 @@ Histogram.prototype.format = function(result, override) { var width = getWidth(override); var binsStart = getBinStart(override); var nulls = 0; + var infinities = 0; + var nans = 0; var avg; + var timestampStart; + var aggregation; + var offset; if (result.rows.length) { var firstRow = result.rows[0]; @@ -249,23 +595,60 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; - binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; + timestampStart = firstRow.timestamp_start; + infinities = firstRow.infinities_count; + nans = firstRow.nans_count; + binsStart = populateBinStart(override, firstRow); + + if (Number.isFinite(timestampStart)) { + aggregation = getAggregation(override, this.aggregation); + offset = getOffset(override, this.offset); + } buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit( + row, + 'bins_number', + 'bin_width', + 'nulls_count', + 'infinities_count', + 'nans_count', + 'avg_val', + 'timestamp_start' + ); }); } return { + aggregation: aggregation, + offset: offset, + timestamp_start: timestampStart, bin_width: width, bins_count: binsCount, bins_start: binsStart, nulls: nulls, + infinities: infinities, + nans: nans, avg: avg, bins: buckets }; }; +function getAggregation(override, aggregation) { + return override && override.aggregation ? override.aggregation : aggregation; +} + +function getOffset(override, offset) { + if (override && override.offset) { + return override.offset; + } + if (offset) { + return offset; + } + + return 0; +} + function getBinStart(override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); @@ -295,6 +678,32 @@ function getWidth(override) { return width; } +function parseOffset(offset, aggregation) { + if (!offset) { + return '0'; + } + if (aggregation === 'hour' || aggregation === 'minute') { + return '0'; + } + + var offsetInHours = Math.ceil(offset / 3600); + return '' + offsetInHours; +} + +function populateBinStart(override, firstRow) { + var binStart; + + if (firstRow.hasOwnProperty('timestamp')) { + binStart = firstRow.timestamp; + } else if (override.hasOwnProperty('start')) { + binStart = getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; +} + Histogram.prototype.getType = function() { return TYPE; }; diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index da63b27f..5df092f4 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,14 +1,36 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../aggregation'); +var debug = require('debug')('windshaft:widget:aggregation:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' sum(_feature_count) AS count,', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -17,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', @@ -36,40 +58,46 @@ var categoriesSummaryCountQueryTpl = dot.template([ ' SELECT count(1) AS categories_count', ' FROM (', ' SELECT {{=it._column}} AS category', - ' FROM ({{=it._query}}) _cdb_categories', + ' FROM filtered_source', ' GROUP BY {{=it._column}}', ' ) _cdb_categories_count', ')' ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', - 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', + 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); var CATEGORIES_LIMIT = 6; - function Aggregation(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -78,27 +106,49 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } var _query = this.rewrittenQuery(this.query); + var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; + + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ "WITH", [ - summaryQueryTpl({ + filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn + }), + summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -110,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -120,15 +171,23 @@ Aggregation.prototype.sql = function(psql, override, callback) { aggregationSql = [ "WITH", [ - summaryQueryTpl({ + filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn + }), + summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -140,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _limit: CATEGORIES_LIMIT @@ -147,6 +207,8 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } + debug(aggregationSql); + return callback(null, aggregationSql); }; diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index 1425e2d1..38b2c119 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -1,14 +1,15 @@ var _ = require('underscore'); var BaseDataview = require('../base'); -function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { +function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) { this.BaseDataview = BaseDataview; this.query = query; this.queryOptions = queryOptions; this.queryRewriter = queryRewriter; this.queryRewriteData = queryRewriteData; this.options = options; - this.baseDataview = new this.BaseDataview(this.query, this.queryOptions); + this.queries = queries; + this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries); } module.exports = BaseOverviewsDataview; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 9e331f0b..64d612c9 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,34 +1,61 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); +var debug = require('debug')('windshaft:widget:formula:overview'); var dot = require('dot'); dot.templateSettings.strip = false; var formulaQueryTpls = { - 'count': dot.template([ - 'SELECT', - 'sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), - 'sum': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), - 'avg': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), + 'count': dot.template([ + 'SELECT', + 'sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'sum': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), + 'avg': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), }; -function Formula(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function Formula(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; + this.queries = queries; } Formula.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -36,21 +63,38 @@ Formula.prototype.constructor = Formula; module.exports = Formula; -Formula.prototype.sql = function(psql, override, callback) { +Formula.prototype.sql = function (psql, override, callback) { + var self = this; var formulaQueryTpl = formulaQueryTpls[this.operation]; - if ( formulaQueryTpl ) { + if (formulaQueryTpl) { // supported formula for use with overviews + if (this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var formulaSql = formulaQueryTpl({ - _query: this.rewrittenQuery(this.query), + _isFloatColumn: this._isFloatColumn, + _query: this.rewrittenQuery(this.query), _operation: this.operation, - _column: this.column + _column: this.column }); + callback = callback || override; + debug(formulaSql); + return callback(null, formulaSql); } + // default behaviour return this.defaultSql(psql, override, callback); }; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 67da4514..6674f6a0 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -1,23 +1,35 @@ var _ = require('underscore'); var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../histogram'); +var debug = require('debug')('windshaft:dataview:histogram:overview'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' {{?it._isFloatColumn}}AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join(' \n')); @@ -26,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join('\n')); @@ -37,7 +49,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -56,7 +68,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -76,11 +88,34 @@ var nullsQueryTpl = dot.template([ ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + 'infinities AS (', + ' SELECT', + ' count(*) AS infinities_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + 'nans AS (', + ' SELECT', + ' count(*) AS nans_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', + ' {{?it._isFloatColumn}}infinities_count,', + ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -90,14 +125,14 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', ' sum(_feature_count) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', + 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + ' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; this.queries = queries; @@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; - -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.rewrittenQuery(this.queries.no_filters) - }); if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction @@ -154,11 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) { return this.defaultSql(psql, override, callback); } + var histogramSql = this._buildQuery(override); + + return callback(null, histogramSql); +}; + +Histogram.prototype._buildQuery = function (override) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; var _query = this.rewrittenQuery(this.query); - var basicsQuery, binsQuery; + filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', + _query: _query, + _column: _column + }); - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { + debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, _column: _column, @@ -177,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -198,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - basicsQuery, - binsQuery, - nullsQueryTpl({ + if (this._columnType === 'float') { + cteSql.push( + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) ].join('\n'); - return callback(null, histogramSql); + debug(histogramSql); + + return histogramSql; }; + +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); +}; + + diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js index 7e3b3161..6ec731f4 100644 --- a/lib/cartodb/models/dataview/overviews/list.js +++ b/lib/cartodb/models/dataview/overviews/list.js @@ -1,8 +1,8 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../list'); -function List(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function List(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); } List.prototype = Object.create(BaseOverviewsDataview.prototype); diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js index 8afdf905..0f7072ae 100644 --- a/lib/cartodb/models/filter/bbox.js +++ b/lib/cartodb/models/filter/bbox.js @@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([ ].join('\n')); var bboxFilterTpl = dot.template( - '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})' + 'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))' ); var LATITUDE_MAX_VALUE = 85.0511287798066; @@ -66,7 +66,8 @@ function getBoundingBoxes(west, south, east, north) { bboxes.push([west, south, east, north]); } else { bboxes.push([west, south, 180, north]); - bboxes.push([-180, south, east % 180, north]); + // here we assume west,east have been adjusted => west >= -180 => east > 180 + bboxes.push([-180, south, east - 360, north]); } return bboxes; diff --git a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js index b59684c7..8cb63f48 100644 --- a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js @@ -115,6 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi } layer.options.sql = analysisSql; layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews)); + layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode); } else { missingNodesErrors.push( new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex) @@ -330,4 +331,13 @@ function AnalysisError(message) { this.message = message; } +function getAllAffectedTablesFromSourceNodes(node) { + var affectedTables = node.getAllInputNodes(function (node) { + return node.getType() === 'source'; + }).reduce(function(list, node) { + return list.concat(node.getAffectedTables()); + },[]); + return affectedTables; +} + require('util').inherits(AnalysisError, Error); diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js new file mode 100644 index 00000000..aead2d91 --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -0,0 +1,25 @@ +function MapConfigBufferSizeAdapter() { + this.formats = ['png', 'png32', 'mvt', 'grid.json']; +} + +module.exports = MapConfigBufferSizeAdapter; + +MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { + if (!context.templateParams || !context.templateParams.buffersize) { + return callback(null, requestMapConfig); + } + + this.formats.forEach(function (format) { + if (Number.isFinite(context.templateParams.buffersize[format])) { + if (requestMapConfig.buffersize === undefined) { + requestMapConfig.buffersize = {}; + } + + requestMapConfig.buffersize[format] = context.templateParams.buffersize[format]; + } + }); + + setImmediate(function () { + callback(null, requestMapConfig); + }); +}; diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js index fd00e344..17c2059f 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js @@ -43,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC if (nestedNamedLayers.length > 0) { var nestedNamedMapsError = new Error('Nested named layers are not allowed'); - // nestedNamedMapsError.http_status = 400; return done(nestedNamedMapsError); } diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index 46efd100..b9487095 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -4,13 +4,6 @@ var dot = require('dot'); dot.templateSettings.strip = false; var queue = require('queue-async'); var PSQL = require('cartodb-psql'); -/** - * cartodb-psql creates `global.Promise` as an empty constructor. - * However, `turbo-carto` relies on a polyfil that fails to create the polyfil - * as it finds `global.Promise` but it doesn't find `Promise.resolve`. - */ -global.Promise = global.Promise || function() {}; -global.Promise.resolve = global.Promise.resolve || function() {}; var turboCarto = require('turbo-carto'); var SubstitutionTokens = require('../../../utils/substitution-tokens'); diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index af5a775b..340073b5 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { var context = {}; step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, this); + self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index c07f9abc..177322d4 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var context = {}; step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, this); + self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 2439a650..17c66c29 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -90,6 +90,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { }, function instantiateTemplate(err, templateParams) { assert.ifError(err); + context.templateParams = templateParams; return self.templateMaps.instance(self.template, templateParams); }, function prepareAdapterMapConfig(err, requestMapConfig) { @@ -113,7 +114,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { function prepareContextLimits(err, _mapConfig) { assert.ifError(err); mapConfig = _mapConfig; - self.userLimitsApi.getRenderLimits(self.owner, this); + self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this); }, function cacheAndReturnMapConfig(err, renderLimits) { self.err = err; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index b695a367..350e52a0 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -4,6 +4,8 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var _ = require('underscore'); +var lzmaMiddleware = require('./middleware/lzma'); + var controller = require('./controllers'); var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); @@ -35,12 +37,15 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter'); +var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter'); var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter'); var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter'); var MapConfigAdapter = require('./models/mapconfig/adapter'); +var StatsBackend = require('./backends/stats'); + module.exports = function(serverOptions) { // Make stats client globally accessible global.statsClient = StatsClient.getInstance(serverOptions.statsd); @@ -115,8 +120,27 @@ module.exports = function(serverOptions) { var onTileErrorStrategy; if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { - if (err && err.message === 'Render timed out' && format === 'png') { - return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {}); + + function isRenderTimeoutError (err) { + return err.message === 'Render timed out'; + } + + function isDatasourceTimeoutError (err) { + return err.message && err.message.match(/canceling statement due to statement timeout/i); + } + + function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); + } + + function isRasterFormat (format) { + return format === 'png' || format === 'jpg'; + } + + if (isTimeoutError(err) && isRasterFormat(format)) { + return callback(null, timeoutErrorTile, { + 'Content-Type': 'image/png', + }, {}); } else { return callback(err, tile, headers, stats); } @@ -150,11 +174,14 @@ module.exports = function(serverOptions) { var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); + var statsBackend = new StatsBackend(); + var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; var mapConfigAdapter = new MapConfigAdapter( new MapConfigNamedLayersAdapter(templateMaps, pgConnection), + new MapConfigBufferSizeAdapter(), new SqlWrapMapConfigAdapter(), new DataviewsWidgetsAdapter(), new AnalysisMapConfigAdapter(analysisBackend), @@ -207,7 +234,8 @@ module.exports = function(serverOptions) { surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache, - mapConfigAdapter + mapConfigAdapter, + statsBackend ).register(app); new controller.NamedMaps( @@ -303,6 +331,25 @@ function bootstrap(opts) { app.enable('jsonp callback'); app.disable('x-powered-by'); app.disable('etag'); + + // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 + // See: http://expressjs.com/en/4x/api.html#app.set + app.set('json replacer', function (key, value) { + if (value !== value) { + return 'NaN'; + } + + if (value === Infinity) { + return 'Infinity'; + } + + if (value === -Infinity) { + return '-Infinity'; + } + + return value; + }); + app.use(bodyParser.json()); app.use(function bootstrap$prepareRequestResponse(req, res, next) { @@ -319,6 +366,8 @@ function bootstrap(opts) { next(); }); + app.use(lzmaMiddleware); + // temporary measure until we upgrade to newer version expressjs so we can check err.status app.use(function(err, req, res, next) { if (err) { diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js new file mode 100644 index 00000000..47d730f4 --- /dev/null +++ b/lib/cartodb/utils/query-utils.js @@ -0,0 +1,26 @@ +function prepareQuery(sql) { + var affectedTableRegexCache = { + bbox: /!bbox!/g, + scale_denominator: /!scale_denominator!/g, + pixel_width: /!pixel_width!/g, + pixel_height: /!pixel_height!/g + }; + + return sql + .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') + .replace(affectedTableRegexCache.scale_denominator, '0') + .replace(affectedTableRegexCache.pixel_width, '1') + .replace(affectedTableRegexCache.pixel_height, '1'); +} + +module.exports.extractTableNames = function extractTableNames(query) { + return [ + 'SELECT * FROM CDB_QueryTablesText($windshaft$', + prepareQuery(query), + '$windshaft$) as tablenames' + ].join(''); +}; + +module.exports.getQueryRowCount = function getQueryRowEstimation(query) { + return 'select CDB_EstimateRowCount(\'' + query + '\') as rows'; +}; diff --git a/package.json b/package.json index 813c943d..02c94e52 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.1.2", + "version": "3.12.11", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -16,14 +16,17 @@ "contributors": [ "Simon Tokumine ", "Javi Santana ", - "Sandro Santilli " + "Sandro Santilli ", + "Carlos Matallín ", + "Daniel Garcia Aubert ", + "Mario de Frutos " ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.50.3", - "cartodb-psql": "~0.7.1", + "camshaft": "0.58.1", + "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.2.0", - "cartodb-redis": "0.13.2", + "cartodb-redis": "0.14.0", "debug": "~2.2.0", "dot": "~1.0.2", "express": "~4.13.3", @@ -38,20 +41,22 @@ "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.19.0", + "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.0.1", + "windshaft": "3.3.2", "yargs": "~5.0.0" }, "devDependencies": { "istanbul": "~0.4.3", - "jshint": "~2.6.0", - "mocha": "~1.21.4", + "jshint": "~2.9.4", + "mocha": "~3.4.1", + "moment": "~2.18.1", "nock": "~2.11.0", "redis": "~0.12.1", "strftime": "~0.8.2" }, "scripts": { + "lint": "jshint lib test", "preinstall": "make pre-install", "test": "make test-all" }, diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 83c2f535..2f087626 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client'); var dot = require('dot'); var debug = require('debug')('windshaft:cartodb:test'); -describe('analysis-layers use cases', function() { +describe('analysis-layers use cases', function () { - var multitypeStyleTemplate = dot.template([ - "#points['mapnik::geometry_type'=1] {", - " marker-fill-opacity: {{=it._opacity}};", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: {{=it._opacity}};", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: {{=it._color}};", - " marker-allow-overlap: true;", - "}", - "#lines['mapnik::geometry_type'=2] {", - " line-color: {{=it._color}};", - " line-width: 2;", - " line-opacity: {{=it._opacity}};", - "}", - "#polygons['mapnik::geometry_type'=3] {", - " polygon-fill: {{=it._color}};", - " polygon-opacity: {{=it._opacity}};", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: {{=it._opacity}};", - "}" - ].join('\n')); + var multitypeStyleTemplate = dot.template( + `#points['mapnik::geometry_type'=1] { + marker-fill-opacity: {{=it._opacity}}; + marker-line-color: #FFF; + marker-line-width: 0.5; + marker-line-opacity: {{=it._opacity}}; + marker-placement: point; + marker-type: ellipse; + marker-width: 8; + marker-fill: {{=it._color}}; + marker-allow-overlap: true; + } + #lines['mapnik::geometry_type'=2] { + line-color: {{=it._color}}; + line-width: 2; + line-opacity: {{=it._opacity}}; + } + #polygons['mapnik::geometry_type'=3] { + polygon-fill: {{=it._color}}; + polygon-opacity: {{=it._opacity}}; + line-color: #FFF; + line-width: 0.5; + line-opacity: {{=it._opacity}}; + }` + ); function cartocss(color, opacity) { @@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() { version: '1.5.0', layers: layers, dataviews: dataviews || {}, - analysis: analysis || [] + analyses: analysis || [] }; } - function analysisDef(analysis) { - return JSON.stringify(analysis); - } - var DEFAULT_MULTITYPE_STYLE = cartocss(); var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 }; + var pointInPolygonDef = { + id: 'a1', + type: 'point-in-polygon', + params: { + points_source: { + type: 'source', + params: { + query: 'select * from analysis_rent_listings' + } + }, + polygons_source: { + type: 'buffer', + params: { + source: { + type: 'source', + params: { + query: 'select * from analysis_banks' + } + }, + radius: 250 + } + } + } + }; + + var bufferDef = { + id: 'b1', + type: 'buffer', + params: { + source: { + type: 'source', + params: { + query: 'select * from analysis_banks' + } + }, + radius: 250 + } + }; + var useCases = [ { desc: '1 mapnik layer', @@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } @@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_banks", + sql: 'select * from analysis_banks', cartocss: cartocss('#2167AB'), cartocss_version: '2.3.0' } @@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } @@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } }, { - type: 'analysis', + type: 'cartodb', options: { - def: analysisDef({ - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - }), - cartocss: cartocss('black', 0.5) + source: { + id: 'b1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } - ]) + ], + {}, + [ + bufferDef + ] + ) }, { @@ -137,531 +169,115 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } }, { - type: 'analysis', + type: 'cartodb', options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - } - } - }), - cartocss: cartocss('green', 1.0) + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } - ]) + ], + {}, + [ + pointInPolygonDef + ] + ) }, { desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - }, - { - type: 'cartodb', - options: { - sql: "select * from analysis_rent_listings", - cartocss: DEFAULT_MULTITYPE_STYLE, - cartocss_version: '2.3.0' - } - } - ]) - }, - - { - desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'cartodb', - options: { - sql: "select * from analysis_rent_listings", - cartocss: DEFAULT_MULTITYPE_STYLE, - cartocss_version: '2.3.0' - } - }, - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 300 - } - }), - cartocss: cartocss('magenta', 0.5) - } - }, - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 300 - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - } - ]) - }, - - { - skip: true, - desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'cartodb', - options: { - "source": { id: "a" }, - "cartocss": DEFAULT_MULTITYPE_STYLE, - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "b1" }, - "cartocss": cartocss('green', 1.0), - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "b2" }, - "cartocss": cartocss('magenta', 0.5), - "cartocss_version": "2.3.0" - } - } - ], - [ - { - id: "b2", - options: { - def: analysisDef({ - "type": "count-in-polygon", - "id": "a0", - "params": { - "columnName": 'count_airbnb', - "pointsSource": { - "type": "source", - "params": { - query: "select * from analysis_rent_listings" - }, - dataviews: { - price_histogram: { - type: 'histogram', - options: { - column: 'price' - } - } - } - }, - "polygonsSource": { - "id": "b1", - "type": "buffer", - "params": { - "source": { - "id": "b0", - "type": "source", - "params": { - query: "select * from analysis_banks" - } - }, - "radius": 250 - }, - dataviews: { - bank_category: { - type: 'aggregation', - options: { - column: 'bank' - } - } - } - } - }, - dataviews: { - count_histogram: { - type: 'histogram', - options: { - column: 'count_airbnb' - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - } - ]) - }, - - { - skip: true, - desc: 'I. Distribution centers', mapConfig: mapConfig( - // layers [ { type: 'cartodb', options: { - "source": { id: "b0" }, - "cartocss": [ - "#distribution_centers {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 0.7;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: blue;", - " marker-allow-overlap: true;", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } }, { type: 'cartodb', options: { - "source": { id: "a0" }, - "cartocss": [ - "#shops {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 0.7;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: red;", - " marker-allow-overlap: true;", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "a1" }, - "cartocss": [ - "#routing {", - " line-color: ramp([routing_time], colorbrewer(Reds));", - " line-width: ramp([routing_time], 2, 8);", - " line-opacity: 1.0;", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" + sql: 'select * from analysis_rent_listings', + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } ], - // dataviews - { - distribution_center_name_category: { - source: { id: 'b0' }, - type: 'aggregation', - options: { - column: 'name' - } - }, - time_histogram: { - source: { id: 'a1' }, - type: 'histogram', - options: { - column: 'routing_time' - } - }, - distance_histogram: { - source: { id: 'a1' }, - type: 'histogram', - options: { - column: 'routing_distance' - } - } - }, - // analysis + {}, [ - { - id: 'a1', - type: 'routing-n-to-n', - params: { - // distanceColumn: 'routing_distance', - // timeColumn: 'routing_time', - originSource: { - id: 'b0', - type: 'source', - params: { - query: 'select * from distribution_centers' - } - }, - destinationSource: { - id: 'a0', - type: 'source', - params: { - query: 'select * from shops' - } - } - } - } + pointInPolygonDef ] ) }, { - skip: true, - desc: 'II. Population analysis', + desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', mapConfig: mapConfig( - // layers - [ - { - type: 'cartodb', - options: { - "source": { id: "a2" }, - "cartocss": [ - "#count_in_polygon {", - " polygon-opacity: 1.0", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: 0.7", - " polygon-fill: ramp([estimated_people], colorbrewer(Reds));", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "a0" }, - "cartocss": DEFAULT_MULTITYPE_STYLE, - "cartocss_version": "2.3.0" - } - } - ], - // dataviews - { - total_population_formula: { - "source": { id: "a3" }, - type: 'formula', - options: { - column: 'total_population', - operation: 'sum' - } - }, - people_histogram: { // this injects a range filter at `a2` node output - "source": { id: "a2" }, - type: 'histogram', - options: { - column: 'estimated_people' - } - }, - subway_line_category: { // this injects a category filter at `a0` node output - "source": { id: "a0" }, - type: 'aggregation', - options: { - column: 'subway_line' - } - } - }, - // analysis - [ - { - id: 'a3', - // this will union the polygons, produce just one polygon, and calculate the total population for it - type: 'total-population', - params: { - columnName: 'total_population', - source: { - id: 'a2', - type: 'estimated-population', - params: { - columnName: 'estimated_people', - source: { - id: 'a1', - type: 'trade-area', - params: { - source: { - "id": "a0", - "type": "source", - "params": { - query: "select * from subway_stops" - } - }, - kind: 'walk', - time: 300 - } - } - } - } - } - } - ]) - }, - - { - skip: true, - desc: 'III. Point in polygon', - mapConfig: mapConfig( - // layers [ { type: 'cartodb', options: { - "source": { id: "a1" }, - "cartocss": [ - "#count_in_polygon {", - " polygon-opacity: 1.0", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: 0.7", - " polygon-fill: ramp([count_people], colorbrewer(Reds));", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" + sql: 'select * from analysis_rent_listings', + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + source: { + id: 'b1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } ], - // dataviews - { - age_histogram: { - "source": { id: "a0" }, - type: 'histogram', - options: { - column: 'age' - } - }, - income_histogram: { - "source": { id: "a0" }, - type: 'histogram', - options: { - column: 'income' - } - }, - gender_category: { - "source": { id: "a0" }, - type: 'aggregation', - options: { - column: 'gender' - } - } - }, - // analysis + {}, [ - { - "id": "a1", - "type": "count-in-polygon", - "params": { - "columnName": 'count_people', - "pointsSource": { - "id": 'a0', - "type": "source", - "params": { - query: "select the_geom, age, gender, income from people" - } - }, - "polygonsSource": { - "id": "b0", - "type": "source", - "params": { - query: "select * from postal_codes" - } - } - } - } + bufferDef, + pointInPolygonDef ] ) } - ]; - useCases.forEach(function(useCase, imageIdx) { + useCases.forEach(function (useCase) { if (!!useCase.skip) { - debug(JSON.stringify(useCase.mapConfig, null, 4)); + return debug(JSON.stringify(useCase.mapConfig, null, 4)); } - it.skip('should implement use case: "' + useCase.desc + '"', function(done) { + it(`should implement use case: '${useCase.desc}'`, function (done) { var testClient = new TestClient(useCase.mapConfig, 1234); var tile = useCase.tile || TILE_ANALYSIS_TABLES; - testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { + testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) { assert.ok(!err, err); - image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); + //image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); assert.equal(image.width(), 256); diff --git a/test/acceptance/analysis/error-cases.js b/test/acceptance/analysis/error-cases.js index 3bde8442..b79e1046 100644 --- a/test/acceptance/analysis/error-cases.js +++ b/test/acceptance/analysis/error-cases.js @@ -373,5 +373,70 @@ describe('analysis-layers error cases', function() { }); }); + it('should return "function does not exist" indicating the node_id and context', function(done) { + var mapConfig = createMapConfig([{ + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": '#polygons { polygon-fill: red; }', + "cartocss_version": "2.3.0" + } + }], {}, [{ + "id": "HEAD", + "type": "buffer", + "params": { + "source": { + "id": "HEAD2", + "type": "buffer", + "params": { + "source": { + "id": "HEAD3", + "type": 'deprecated-sql-function', + "params": { + "id": "HEAD4", + "function_name": 'DEP_EXT_does_not_exist_fn', + "primary_source": { + "type": 'source', + "params": { + "query": "select * from populated_places_simple_reduced" + } + }, + "function_args": ['wadus'] + } + }, + "radius": 10 + } + }, + "radius": 10 + } + }]); + + var testClient = new TestClient(mapConfig, 1234); + + testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + assert.ok(!err, err); + + assert.equal(layergroupResult.errors.length, 1); + assert.equal( + layergroupResult.errors[0], + 'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist' + ); + + assert.equal(layergroupResult.errors_with_context[0].type, 'analysis'); + assert.equal( + layergroupResult.errors_with_context[0].message, + 'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist' + ); + assert.equal(layergroupResult.errors_with_context[0].analysis.id, 'HEAD'); + assert.equal(layergroupResult.errors_with_context[0].analysis.type, 'buffer'); + assert.equal(layergroupResult.errors_with_context[0].analysis.node_id, 'HEAD3'); + + testClient.drain(done); + }); + }); + + }); diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js new file mode 100644 index 00000000..c32f62d1 --- /dev/null +++ b/test/acceptance/buffer-size-format.js @@ -0,0 +1,441 @@ +require('../support/test_helper'); + +var fs = require('fs'); +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); +var mapnik = require('windshaft').mapnik; +var IMAGE_TOLERANCE_PER_MIL = 5; + +var CARTOCSS_LABELS = [ + '#layer {', + ' polygon-fill: #374C70;', + ' polygon-opacity: 0.9;', + ' line-width: 1;', + ' line-color: #FFF;', + ' line-opacity: 0.5;', + '}', + '#layer::labels {', + ' text-name: [name];', + ' text-face-name: \'DejaVu Sans Book\';', + ' text-size: 20;', + ' text-fill: #FFFFFF;', + ' text-label-position-tolerance: 0;', + ' text-halo-radius: 1;', + ' text-halo-fill: #6F808D;', + ' text-dy: -10;', + ' text-allow-overlap: true;', + ' text-placement: point;', + ' text-placement-type: dummy;', + '}' +].join('\n'); + +function createMapConfig (bufferSize, cartocss) { + cartocss = cartocss || CARTOCSS_LABELS; + + return { + version: '1.6.0', + buffersize: bufferSize, + layers: [{ + type: "cartodb", + options: { + sql: [ + 'select', + ' *', + 'from', + ' populated_places_simple_reduced', + ].join('\n'), + cartocss: cartocss, + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + }; +} + +describe('buffer size per format', function () { + var testCases = [ + { + desc: 'should get png tile using buffer-size 0', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + mapConfig: createMapConfig({ png: 0, 'grid.json': 0 }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + mapConfig: createMapConfig({ png: 128, 'grid.json': 128 }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get mvt tile using buffer-size 0', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.mvt', + mapConfig: createMapConfig({ mvt: 0 }), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features; + assert.equal(features.length, 1); + callback(); + } + }, + { + desc: 'should get mvt tile using buffer-size 128', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.mvt', + mapConfig: createMapConfig({ mvt: 128 }), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features; + assert.equal(features.length, 9); + callback(); + } + }, + { + desc: 'should get grid.json tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + layers: [0], + fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', + mapConfig: createMapConfig({ 'grid.json': 0 }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback); + } + }, + { + desc: 'should get grid.json tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + layers: [0], + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', + mapConfig: createMapConfig({ 'grid.json': 128 }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); + } + } + ]; + + testCases.forEach(function (test) { + it(test.desc, function (done) { + var testClient = new TestClient(test.mapConfig, 1234); + var coords = test.coords; + var options = { + format: test.format, + layers: test.layers + }; + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + // tile.save(test.fixturePath); + test.assert(tile, function (err) { + assert.ifError(err); + testClient.drain(done); + }); + }); + }); + }); +}); + +function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) { + cartocss = cartocss || CARTOCSS_LABELS; + + return { + "version": "0.0.1", + "name": name, + "placeholders": placeholders || { + "buffersize": { + "type": "number", + "default": 0 + } + }, + "layergroup": createMapConfig(buffersize) + }; +} + +describe('buffer size per format for named maps', function () { + var testCases = [ + { + desc: 'should get png tile using buffer-size 0 (default value in template)', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128 (placehoder value)', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { buffersize: 128 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + template: createBufferSizeTemplate('named-custom-buffer-size', { png: '<%= buffersize %>'}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 0 (default value in template by format)', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { buffersize_png: 0 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-default-buffer-size-by-format', { + png: '<%= buffersize_png %>' + }, { + "buffersize_png": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { buffersize_png: 128 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + template: createBufferSizeTemplate('named-custom-buffer-size-by-format', { + png: '<%= buffersize_png %>' + }, { + "buffersize_png": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get grid.json tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + layers: [0], + placeholders: { buffersize_gridjson: 0 }, + fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', + template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', { + 'grid.json': '<%= buffersize_gridjson %>' + }, { + "buffersize_gridjson": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback); + } + }, + { + desc: 'should get grid.json tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + layers: [0], + placeholders: { buffersize_gridjson: 128 }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', + template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', { + 'grid.json': '<%= buffersize_gridjson %>' + }, { + "buffersize_gridjson": { + "type": "number", + "default": "0" + } + }), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); + } + } + ]; + + testCases.forEach(function (test) { + it(test.desc, function (done) { + var testClient = new TestClient(test.template, 1234); + var coords = test.coords; + var options = { + format: test.format, + placeholders: test.placeholders, + layers: test.layers + }; + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png'); + test.assert(tile, function (err) { + assert.ifError(err); + testClient.drain(done); + }); + }); + }); + }); +}); + + +describe('buffer size per format for named maps w/o placeholders', function () { + var testCases = [ + { + desc: 'should get png tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { + buffersize: { + png: 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-no-buffer-size-png-0', {}, {}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get png tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { + buffersize: { + png: 128 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png', + template: createBufferSizeTemplate('named-no-buffer-size-png-128', {}, {}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + { + desc: 'should get mvt tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + placeholders: { + buffersize: { + mvt: 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt', + template: createBufferSizeTemplate('named-no-buffer-size-mvt', {}, {}), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features; + + var dataFixture = fs.readFileSync(this.fixturePath); + var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y); + vtile.setDataSync(dataFixture); + var vtileJSON = vtile.toJSON(); + var vtileFeatures = vtileJSON[0].features; + + assert.equal(features.length, vtileFeatures.length); + callback(); + } + }, + { + desc: 'should get mvt tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'mvt', + placeholders: { + buffersize: { + mvt: 128 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt', + template: createBufferSizeTemplate('named-no-buffer-size-mvt-128', {}, {}), + assert: function (tile, callback) { + var tileJSON = tile.toJSON(); + var features = tileJSON[0].features; + + var dataFixture = fs.readFileSync(this.fixturePath); + var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y); + vtile.setDataSync(dataFixture); + var vtileJSON = vtile.toJSON(); + var vtileFeatures = vtileJSON[0].features; + + assert.equal(features.length, vtileFeatures.length); + callback(); + } + }, + { + desc: 'should get grid.json tile using buffer-size 0 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + layers: [0], + placeholders: { + buffersize: { + 'grid.json': 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json', + template: createBufferSizeTemplate('named-no-buffer-size-grid-json-0', {}, {}), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback); + } + }, + { + desc: 'should get grid.json tile using buffer-size 128 overriden by template params', + coords: { z: 7, x: 64, y: 48 }, + format: 'grid.json', + layers: [0], + placeholders: { + buffersize: { + 'grid.json': 128 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json', + template: createBufferSizeTemplate('named-no-buffer-size-grid-json-128', {}, {}), + assert: function (tile, callback) { + assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback); + } + }, + { + desc: 'should get png tile using buffer-size 0' + + ' overriden by template params with no buffersize in mapconfig', + coords: { z: 7, x: 64, y: 48 }, + format: 'png', + placeholders: { + buffersize: { + png: 0 + } + }, + fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png', + template: createBufferSizeTemplate('named-no-buffer-size-mapconfig-png-0', undefined, {}), + assert: function (tile, callback) { + assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback); + } + }, + + ]; + + testCases.forEach(function (test) { + it(test.desc, function (done) { + var testClient = new TestClient(test.template, 1234); + var coords = test.coords; + var options = { + format: test.format, + placeholders: test.placeholders, + layers: test.layers + }; + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + //tile.save(test.fixturePath); + // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); + // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); + test.assert(tile, function (err) { + assert.ifError(err); + testClient.drain(done); + }); + }); + }); + }); +}); diff --git a/test/acceptance/cache/cache_headers.js b/test/acceptance/cache/cache_headers.js new file mode 100644 index 00000000..2cd916af --- /dev/null +++ b/test/acceptance/cache/cache_headers.js @@ -0,0 +1,393 @@ +var testHelper = require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var qs = require('querystring'); + +var CartodbWindshaft = require('../../../lib/cartodb/server'); +var serverOptions = require('../../../lib/cartodb/server_options'); +var server = new CartodbWindshaft(serverOptions); +server.setMaxListeners(0); + +var LayergroupToken = require('../../support/layergroup-token'); + +describe('get requests with cache headers', function() { + + var keysToDelete; + beforeEach(function() { + keysToDelete = {}; + }); + + afterEach(function(done) { + testHelper.deleteRedisKeys(keysToDelete, done); + }); + + var statusOkResponse = { + status: 200 + }; + + var mapConfigs = [ + { + "description": "cache headers should be present", + "cache_headers": { + "x_cache_channel": { + "db_name": "test_windshaft_cartodb_user_1_db", + "tables": ["public.test_table"] + }, + "surrogate_keys": "t:77pJnX" + }, + "data": + { + version: '1.5.0', + layers: [ + { + options: { + source: { + id: "2570e105-7b37-40d2-bdf4-1af889598745" + }, + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; }', + cartocss_version: '2.3.0', + attributes: { + id:'cartodb_id', + columns: [ + 'name', + 'address' + ] + } + } + } + ], + analyses: [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from test_table limit 2" + } + } + ] + }, + }, + { + "description": "cache headers should be present and be composed with source table name", + "cache_headers": { + "x_cache_channel": { + "db_name": "test_windshaft_cartodb_user_1_db", + "tables": ["public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d", + "public.test_table"] + }, + "surrogate_keys": "t:77pJnX t:iL4eth" + }, + "data": + { + version: '1.5.0', + layers: [ + { + options: { + source: { + id: "2570e105-7b37-40d2-bdf4-1af889598745" + }, + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; }', + cartocss_version: '2.3.0', + attributes: { + id:'cartodb_id', + columns: [ + 'name', + 'address' + ] + } + } + } + ], + analyses: [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from test_table limit 2" + } + }, + "radius": 50000 + } + } + ] + } + }]; + + var layergroupRequest = function(mapConfig) { + return { + url: '/api/v1/map?api_key=1234&config=' + encodeURIComponent(JSON.stringify(mapConfig)), + method: 'GET', + headers: { + host: 'localhost' + } + }; + }; + + function getRequest(url, addApiKey, callbackName) { + var params = {}; + if (!!addApiKey) { + params.api_key = '1234'; + } + if (!!callbackName) { + params.callback = callbackName; + } + + return { + url: url + '?' + qs.stringify(params), + method: 'GET', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } + }; + } + + function validateCacheHeaders(done, expectedCacheHeaders) { + return function(res, err) { + if (err) { + return done(err); + } + + assert.ok(res.headers['x-cache-channel']); + assert.ok(res.headers['surrogate-key']); + if (expectedCacheHeaders) { + validateXChannelHeaders(res.headers, expectedCacheHeaders); + assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys); + } + + done(); + }; + } + + function validateXChannelHeaders(headers, expectedCacheHeaders) { + var dbName = headers['x-cache-channel'].split(':')[0]; + var tables = headers['x-cache-channel'].split(':')[1].split(',').sort(); + assert.equal(dbName, expectedCacheHeaders.x_cache_channel.db_name); + assert.deepEqual(tables, expectedCacheHeaders.x_cache_channel.tables.sort()); + } + + function noCacheHeaders(done) { + return function(res, err) { + if (err) { + return done(err); + } + + assert.ok( + !res.headers['x-cache-channel'], + 'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`' + ); + assert.ok( + !res.headers['surrogate-key'], + 'did not expect surrogate-key header, got: `' + res.headers['surrogate-key'] + '`' + ); + done(); + }; + } + + function withLayergroupId(mapConfig, callback) { + assert.response( + server, + layergroupRequest(mapConfig), + statusOkResponse, + function(res, err) { + if (err) { + return callback(err); + } + var layergroupId = JSON.parse(res.body).layergroupid; + keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + callback(null, layergroupId, res); + } + ); + } + + mapConfigs.forEach(function(mapConfigData) { + describe(mapConfigData.description, function() { + var mapConfig = mapConfigData.data; + var expectedCacheHeaders = mapConfigData.cache_headers; + it('/api/v1/map Map instantiation', function(done) { + var testFn = validateCacheHeaders(done, expectedCacheHeaders); + withLayergroupId(mapConfig, function(err, layergroupId, res) { + testFn(res); + }); + }); + + it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true), + validateCacheHeaders(done, expectedCacheHeaders) + ); + }); + }); + + it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true), + validateCacheHeaders(done, expectedCacheHeaders) + ); + }); + }); + + it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true), + validateCacheHeaders(done, expectedCacheHeaders) + ); + }); + }); + + it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true), + validateCacheHeaders(done, expectedCacheHeaders) + ); + }); + }); + + it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true), + validateCacheHeaders(done, expectedCacheHeaders) + ); + }); + }); + + it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) { + withLayergroupId(mapConfig, function(err, layergroupId) { + assert.response( + server, + getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true), + validateCacheHeaders(done, expectedCacheHeaders) + ); + }); + }); + }); + }); + + describe('cache headers should NOT be present', function() { + + it('/', function(done) { + assert.response( + server, + getRequest('/'), + statusOkResponse, + noCacheHeaders(done) + ); + }); + + it('/version', function(done) { + assert.response( + server, + getRequest('/version'), + statusOkResponse, + noCacheHeaders(done) + ); + }); + + it('/health', function(done) { + assert.response( + server, + getRequest('/health'), + statusOkResponse, + noCacheHeaders(done) + ); + }); + + it('/api/v1/map/named list named maps', function(done) { + assert.response( + server, + getRequest('/api/v1/map/named', true), + statusOkResponse, + noCacheHeaders(done) + ); + }); + + describe('with named maps', function() { + + var templateName = 'x_cache'; + + beforeEach(function(done) { + var template = { + version: '0.0.1', + name: templateName, + auth: { + method: 'open' + }, + layergroup: mapConfigs[0].data + }; + + var namedMapRequest = { + url: '/api/v1/map/named?api_key=1234', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(template) + }; + + assert.response( + server, + namedMapRequest, + statusOkResponse, + function(res, err) { + done(err); + } + ); + }); + + afterEach(function(done) { + assert.response( + server, + { + url: '/api/v1/map/named/' + templateName + '?api_key=1234', + method: 'DELETE', + headers: { + host: 'localhost' + } + }, + { + status: 204 + }, + function(res, err) { + done(err); + } + ); + }); + + + it('/api/v1/map/named/:template_id Named map retrieval', function(done) { + assert.response( + server, + getRequest('/api/v1/map/named/' + templateName, true), + statusOkResponse, + noCacheHeaders(done) + ); + }); + + it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) { + assert.response( + server, + getRequest('/api/v1/map/named/' + templateName, true, 'cb'), + statusOkResponse, + noCacheHeaders(done) + ); + }); + }); + }); +}); diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 259cd2af..d8d03177 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -145,4 +145,182 @@ describe('aggregations happy cases', function() { }); }); }); + + var widgetSearchExpects = { + 'count': [ { category: 'other_a', value: 3 } ], + 'sum': [ { category: 'other_a', value: 6 } ], + 'avg': [ { category: 'other_a', value: 2 } ], + 'max': [ { category: 'other_a', value: 3 } ], + 'min': [ { category: 'other_a', value: 1 } ] + }; + + Object.keys(operations_and_values).forEach(function (operation) { + var description = 'should search OTHER category using "' + operation + '"'; + + it(description, function (done) { + this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val')); + this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) { + assert.ifError(err); + + assert.ok(searchResult); + assert.equal(searchResult.type, 'aggregation'); + + assert.equal(searchResult.categories.length, 1); + assert.deepEqual( + searchResult.categories, + widgetSearchExpects[operation] + ); + done(); + }); + }); + }); +}); + +describe('aggregation-dataview: special float values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_aggregation: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'avg', + aggregationColumn: 'val' + } + }, + sum_aggregation_numeric: { + source: { + id: 'a1' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + }, { + "id": "a1", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 3 = 0 THEN \'NaN\'::numeric', + ' WHEN x % 3 = 1 THEN x', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + // Source a0 + // ----------------------------------------------- + // the_geom_webmercator | val | cat + // ----------------------+-----------+------------ + // | -Infinity | category_2 + // | NaN | category_1 + // | 3 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 7 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 11 | category_2 + // | " | " + + var filters = [{ own_filter: 0 }, {}]; + filters.forEach(function (filter) { + it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + assert.ok(dataview.categories.length === 1); + dataview.categories.forEach(function (category) { + assert.ok(category.category === 'category_2'); + assert.ok(category.value === 501); + }); + done(); + }); + }); + + it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.ok(dataview.nans === 333); + assert.ok(dataview.categories.length === 2); + dataview.categories.forEach(function (category) { + assert.ok(category.value !== null); + }); + done(); + }); + }); + }); }); diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js new file mode 100644 index 00000000..941dc07e --- /dev/null +++ b/test/acceptance/dataviews/formula.js @@ -0,0 +1,80 @@ +require('../../support/test_helper'); +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + +describe('formula-dataview: special float values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_formula: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: 'val', + operation: 'avg' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_formula', {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.result, 501); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index abbaef61..c9e16489 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -2,6 +2,25 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +var moment = require('moment'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} describe('histogram-dataview', function() { @@ -13,15 +32,6 @@ describe('histogram-dataview', function() { } }); - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - var mapConfig = createMapConfig( [ { @@ -89,11 +99,1093 @@ describe('histogram-dataview', function() { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('pop_max_histogram', params, function(err, res) { assert.ok(!err, err); - assert.ok(res.errors); assert.equal(res.errors.length, 1); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); + done(); + }); + }); + +}); + +describe('histogram-dataview for date column type', function() { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "datetime-histogram-source" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + datetime_histogram: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } + }, + datetime_histogram_tz: { + source: { + id: 'datetime-histogram-source-tz' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } + }, + datetime_histogram_automatic: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'year' + } + }, + date_histogram_automatic: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + minute_histogram: { + source: { + id: 'minute-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'minute' + } + } + }, + [ + { + "id": "datetime-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "datetime-histogram-source-tz", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date::date AS d", + "from generate_series(", + "'2007-02-15'::date, '2008-04-09'::date, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "minute-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval", + ") date" + ].join(' ') + } + } + ] + ); + + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'datetime_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'datetime_histogram' + }]; + + dateHistogramsUseCases.forEach(function (test) { + it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { + var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + + this.testClient = new TestClient(mapConfig, 1234); + + this.testClient.getDataview(test.dataviewId, {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override aggregation in weeks ' + test.desc, function (done) { + var params = { + aggregation: 'week' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override start and end ' + test.desc, function (done) { + var params = { + start: 1180659600, // 2007-06-01 01:00:00 UTC => '2007-05-31T21:00:00-04:00' + end: 1193792400 // 2007-10-31 01:00:00 UTC + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) { + var params = { + start: -2145916800, + end: 1009843199 + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + + done(); + }); + }); + + it('should return same histogram ' + test.desc, function (done) { + var params = { + start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum) + end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum) + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) { + assert.ok(!err, err); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) { + assert.ok(!err, err); + + assert.deepEqual(dataview, filteredDataview); + done(); + }); + }); + }); + + + it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) { + var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + var params = { + offset: OFFSET_CEST_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram overriding default offset to UTC/GMT ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870399 // 2008-06-30T23:59:59Z, last bin end + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 6); + assert.equal(dataview.bins_count, dataview.bins.length); + done(); + }); + }); + + it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 7); + assert.ok(dataview.bins_count > dataview.bins.length); + done(); + }); + }); + }); + + it('should find the best aggregation (automatic mode) to build the histogram', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should work with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'year'); + assert.equal(dataview.bins.length, 2); + assert.equal(dataview.bins_count, 2); + done(); + }); + }); + + + it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should not apply offset for a histogram aggregated by minutes', function (done) { + var self = this; + var params = { + offset: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + + self.testClient.getDataview('minute_histogram', {}, function (err, dataview) { + assert.ifError(err); + self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithOffset) { + assert.ifError(err); + + assert.notEqual(dataview.offset, dataviewWithOffset.offset); + dataview.offset = dataviewWithOffset.offset; + assert.deepEqual(dataview, dataviewWithOffset); + done(); + }); + }); + }); + + it('should filter by "start" & "end" for a histogram aggregated by minutes', function (done) { + var self = this; + var paramsWithFilter = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum) + }; + + var paramsWithOffset = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum) + offset: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, dataview) { + assert.ifError(err); + + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, filteredDataview) { + assert.ifError(err); + + assert.deepEqual(dataview, filteredDataview); + + self.testClient.getDataview('minute_histogram', paramsWithOffset, + function (err, filteredWithOffsetDataview) { + assert.ifError(err); + + assert.notEqual(filteredWithOffsetDataview.offset, filteredDataview.offset); + filteredWithOffsetDataview.offset = filteredDataview.offset; + assert.deepEqual(filteredWithOffsetDataview, filteredDataview); + done(); + }); + }); + }); + }); + + + it('should return an histogram aggregated by days', function (done) { + var self = this; + var paramsWithDailyAgg = { + aggregation: 'day', + }; + + // data: from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggFixture = { + aggregation: 'day', + bin_width: 600, + bins_count: 2, + bins_start: 1171497600, + timestamp_start: 1171497600, + offset: 0, + nulls: 0, + bins: + [{ + bin: 0, + timestamp: 1171497600, + min: 1171583400, + max: 1171583940, + avg: 1171583670, + freq: 10 + }, + { + bin: 1, + timestamp: 1171584000, + min: 1171584000, + max: 1171584600, + avg: 1171584300, + freq: 11 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAgg, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggFixture); + done(); + }); + }); + + it('should return a histogram aggregated by days with offset', function (done) { + var self = this; + + var paramsWithDailyAggAndOffset = { + aggregation: 'day', + offset: '-3600' + }; + + // data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggAndOffsetFixture = { + aggregation: 'day', + bin_width: 1200, + bins_count: 1, + bins_start: 1171501200, + timestamp_start: 1171497600, + nulls: 0, + offset: -3600, + bins: + [{ + bin: 0, + timestamp: 1171501200, + min: 1171583400, + max: 1171584600, + avg: 1171584000, + freq: 21 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndOffset, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggAndOffsetFixture); + done(); + }); + }); +}); + + +describe('histogram-dataview: special float valuer', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_histogram', {}, function(err, dataview) { + assert.ok(!err, err); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); + +describe('histogram-dates: aggregation input value', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'day' + } + }, + bad_agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'wadus' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'2007-02-15 01:00:00\'::timestamp,', + '\'2008-04-09 01:00:00\'::timestamp,', + ' \'1 day\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should fail when aggregation values is not valid while instantiating the map', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + response: { + status: 400 + } + }; + + this.testClient.getDataview('bad_agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); + + it('should fail when aggregation values is not valid while fetching dataview result', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + aggregation: 'wadus', + response: { + status: 400 + } + }; + + this.testClient.getDataview('agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); +}); + +describe('histogram-dates: timestamp starts at epoch', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + epoch_start_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-04 10:00:00\'::timestamp,', + '\'1984-01-04 10:00:00\'::timestamp,', + ' \'1 month\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should work when timestamp_start is epoch (1970-01-01 = 0)', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = {}; + + this.testClient.getDataview('epoch_start_histogram', override, function(err, dataview) { + assert.ifError(err); + + const { aggregation, timestamp_start } = dataview; + + assert.equal(timestamp_start, 0); + assert.equal(aggregation, 'month'); + + done(); + }); + }); +}); + +describe('histogram-dates: trunc timestamp for each bin respecting user\'s timezone', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + timezone_epoch_histogram_tz: { + source: { + id: 'a1' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + }, + { + id: 'a1', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamptz,', + '\'1970-01-01 01:59:00\'::timestamptz,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'timezone_epoch_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'timezone_epoch_histogram' + }]; + + dateHistogramsUseCases.forEach(function (test) { + it('should return histogram with two buckets ' + test.desc , function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview(test.dataviewId, override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + }); +}); + + +describe('histogram: be able to override with aggregation for histograms instantiated w/o aggregation', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should apply aggregation to the histogram', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + done(); }); }); diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index d23519ae..dcff687f 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() { params: { query: 'select * from test_table_overviews' } + }, + { + id: 'data-source-special-float-values', + type: 'source', + params: { + query: 'select * from test_special_float_values_table_overviews' + } } ], dataviews: { @@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() { aggregationColumn: 'name', } }, + test_categories_special_values: { + type: 'aggregation', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'name', + aggregation: 'sum', + aggregationColumn: 'value', + } + }, test_histogram: { type: 'histogram', source: {id: 'data-source'}, @@ -160,6 +178,16 @@ describe('dataviews using tables with overviews', function() { bins: 2 } }, + test_histogram_special_values: { + type: 'histogram', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + bins: 2 + } + }, test_avg: { type: 'formula', source: {id: 'data-source'}, @@ -168,6 +196,16 @@ describe('dataviews using tables with overviews', function() { operation: 'avg' } }, + test_formula_sum_special_values: { + type: 'formula', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + operation: 'sum' + } + }, test_count: { type: 'formula', source: {id: 'data-source'}, @@ -202,6 +240,17 @@ describe('dataviews using tables with overviews', function() { cartocss_version: '2.3.0', source: { id: 'data-source' } } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_special_float_values_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { + id: 'data-source-special-float-values' + } + } } ] }; @@ -212,7 +261,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -224,7 +280,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":3, + "nulls":0, + "type":"formula", + "infinities": 0, + "nans": 0 + }); testClient.drain(done); }); @@ -236,7 +299,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":5, + "nulls":0, + "type":"formula", + "infinities": 0, + "nans": 0 + }); testClient.drain(done); }); @@ -248,7 +318,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 5, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -260,7 +337,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -275,7 +359,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -372,7 +463,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -383,7 +481,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -395,7 +500,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":1, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -407,7 +519,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -419,7 +538,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -437,7 +563,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -445,5 +578,69 @@ describe('dataviews using tables with overviews', function() { }); + describe('aggregation special float values', function () { + var params = {}; + + it("should expose an aggregation dataview filtering special float values out", function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_categories_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + aggregation: 'sum', + count: 5, + nulls: 0, + nans: 1, + infinities: 1, + min: 6, + max: 6, + categoriesCount: 1, + categories: [ { category: 'Hawai', value: 6, agg: false } ], + type: 'aggregation' + }); + testClient.drain(done); + }); + }); + + it('should expose a histogram dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_histogram_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + bin_width: 0, + bins_count: 1, + bins_start: 3, + nulls: 0, + infinities: 1, + nans: 1, + avg: 3, + bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ], + type: 'histogram' + }); + testClient.drain(done); + }); + }); + + it('should expose a formula (sum) dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + operation: 'sum', + result: 6, + nulls: 0, + nans: 1, + infinities: 1, + type: 'formula' + }); + testClient.drain(done); + }); + }); + }); }); }); diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js deleted file mode 100644 index d0126623..00000000 --- a/test/acceptance/limits.js +++ /dev/null @@ -1,311 +0,0 @@ -var testHelper = require('../support/test_helper'); - -var assert = require('../support/assert'); -var _ = require('underscore'); -var redis = require('redis'); - -var CartodbWindshaft = require('../../lib/cartodb/server'); -var serverOptions = require('../../lib/cartodb/server_options'); - -var LayergroupToken = require('../support/layergroup-token'); - -describe('render limits', function() { - - var layergroupUrl = '/api/v1/map'; - - var redisClient = redis.createClient(global.environment.redis.port); - - var server; - var keysToDelete; - beforeEach(function() { - keysToDelete = {}; - server = new CartodbWindshaft(serverOptions); - server.setMaxListeners(0); - }); - - afterEach(function(done) { - testHelper.deleteRedisKeys(keysToDelete, done); - }); - - var user = 'localhost'; - - var pointSleepSql = "SELECT pg_sleep(0.5)," + - " 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id"; - var pointCartoCss = '#layer { marker-fill:red; }'; - var polygonSleepSql = "SELECT pg_sleep(0.5)," + - " ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id"; - var polygonCartoCss = '#layer { polygon-fill:red; }'; - - function singleLayergroupConfig(sql, cartocss) { - return { - version: '1.0.0', - layers: [ - { - type: 'mapnik', - options: { - sql: sql, - cartocss: cartocss, - cartocss_version: '2.0.1' - } - } - ] - }; - } - - function createRequest(layergroup, userHost) { - return { - url: layergroupUrl, - method: 'POST', - headers: { - host: userHost, - 'Content-Type': 'application/json' - }, - data: JSON.stringify(layergroup) - }; - } - - function withRenderLimit(user, renderLimit, callback) { - redisClient.SELECT(5, function(err) { - if (err) { - return callback(err); - } - var userLimitsKey = 'limits:tiler:' + user; - redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) { - if (err) { - return callback(err); - } - keysToDelete[userLimitsKey] = 5; - return callback(); - }); - }); - - } - - describe('with onTileErrorStrategy DISABLED', function() { - var onTileErrorStrategyEnabled; - before(function() { - onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy; - global.environment.enabledFeatures.onTileErrorStrategy = false; - }); - - after(function() { - global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled; - }); - - it("layergroup creation fails if test tile is slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 400 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, [ 'Render timed out' ]); - done(); - } - ); - }); - }); - - it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) { - withRenderLimit(user, 5000, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.ok(parsed.layergroupid); - keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - done(); - } - ); - }); - }); - - - it("layergroup creation works if test tile is fast but tile request fails if they are slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 400 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed.errors, ['Render timed out']); - done(); - } - ); - - } - ); - }); - }); - - it("tile request does not fail if user limit is high enough", function(done) { - withRenderLimit(user, 5000, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 200, - headers: { - 'Content-Type': 'image/png' - } - }, - function(res, err) { - done(err); - } - ); - - } - ); - }); - }); - - }); - - describe('with onTileErrorStrategy', function() { - - it("layergroup creation works even if test tile is slow", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - var parsed = JSON.parse(res.body); - assert.ok(parsed.layergroupid); - keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - done(); - } - ); - }); - }); - - it("layergroup creation and tile requests works even if they are slow but returns fallback", function(done) { - withRenderLimit(user, 50, function(err) { - if (err) { - return done(err); - } - - var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss); - assert.response(server, - createRequest(layergroup, user), - { - status: 200 - }, - function(res) { - keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - assert.response(server, - { - url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', { - layergroupId: JSON.parse(res.body).layergroupid, - z: 0, - x: 0, - y: 0 - }), - method: 'GET', - headers: { - host: 'localhost' - }, - encoding: 'binary' - }, - { - status: 200, - headers: { - 'Content-Type': 'image/png' - } - }, - function(res, err) { - if (err) { - done(err); - } - var referenceImagePath = './test/fixtures/render-timeout-fallback.png'; - assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25, - function(imgErr/*, similarity*/) { - done(imgErr); - } - ); - } - ); - - } - ); - }); - }); - - }); - -}); diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 275ebab7..210418ac 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1041,7 +1041,7 @@ describe(suiteName, function() { ); }); } - + // See https://github.com/CartoDB/Windshaft-cartodb/issues/91 // and https://github.com/CartoDB/Windshaft-cartodb/issues/38 it("tiles for private tables can be fetched with api_key", function(done) { diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js new file mode 100644 index 00000000..b5a59cd8 --- /dev/null +++ b/test/acceptance/mvt.js @@ -0,0 +1,63 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); + +function createMapConfig (sql = TestClient.SQL.ONE_POINT) { + return { + version: '1.6.0', + layers: [{ + type: "cartodb", + options: { + sql: sql, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + }; +} + +describe('mvt', function () { + const testCases = [ + { + desc: 'should get empty mvt with code 204 (no content)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + response: { + status: 204, + headers: { + 'Content-Type': undefined + } + }, + mapConfig: createMapConfig(TestClient.SQL.EMPTY) + }, + { + desc: 'should get mvt tile with code 200 (ok)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + response: { + status: 200, + headers: { + 'Content-Type': 'application/x-protobuf' + } + }, + mapConfig: createMapConfig() + } + ]; + + testCases.forEach(function (test) { + it(test.desc, done => { + const testClient = new TestClient(test.mapConfig, 1234); + const { z, x, y } = test.coords; + const { format, response } = test; + + testClient.getTile(z, x, y, { format, response }, (err, res) => { + assert.ifError(err); + + assert.equal(res.statusCode, test.response.status); + testClient.drain(done); + }); + }); + }); +}); diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index 91026226..b7af4197 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -21,7 +21,7 @@ describe('named maps static view', function() { var IMAGE_TOLERANCE = 20; - function createTemplate(view) { + function createTemplate(view, layers) { return { version: '0.0.1', name: templateName, @@ -36,7 +36,7 @@ describe('named maps static view', function() { }, view: view, layergroup: { - layers: [ + layers: layers || [ { type: 'mapnik', options: { @@ -192,10 +192,73 @@ describe('named maps static view', function() { } getStaticMap({ zoom: 3 }, function(err, img) { assert.ok(!err); - img.save('/tmp/static.png'); assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done); }); }); }); + it('should return override bbox', function (done) { + var view = { + bounds: { + west: 0, + south: 0, + east: 45, + north: 45 + }, + zoom: 4, + center: { + lng: 40, + lat: 20 + } + }; + templateMaps.addTemplate(username, createTemplate(view), function (err) { + if (err) { + return done(err); + } + getStaticMap({ bbox: '0,45,90,45' }, function(err, img) { + assert.ok(!err); + assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), IMAGE_TOLERANCE, done); + }); + }); + }); + + it('should allow to select the layers to render', function (done) { + var view = { + bounds: { + west: 0, + south: 0, + east: 45, + north: 45 + } + }; + + var layers = [ + { + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer { marker-fill: <%= color %>; }', + cartocss_version: '2.3.0' + } + }, + { + type: 'mapnik', + options: { + sql: 'select ST_Transform(ST_MakeEnvelope(-45, -45, 45, 45, 4326), 3857) the_geom_webmercator', + cartocss: '#layer { polygon-fill: <%= color %>; }', + cartocss_version: '2.3.0' + } + } + ]; + templateMaps.addTemplate(username, createTemplate(view, layers), function (err) { + if (err) { + return done(err); + } + getStaticMap({ layer: 0 }, function(err, img) { + assert.ok(!err); + assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done); + }); + }); + }); + }); diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index f0464699..3408baa2 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -5,6 +5,7 @@ var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); +var TestClient = require('../../support/test-client'); var BaseController = require('../../../lib/cartodb/controllers/base'); @@ -23,6 +24,14 @@ describe('multilayer error cases', function() { BaseController.prototype.req2params = req2paramsFn; }); + // var client = null; + afterEach(function(done) { + if (this.client) { + return this.client.drain(done); + } + return done(); + }); + it("post layergroup with wrong Content-Type", function(done) { assert.response(server, { url: '/database/windshaft_test/layergroup', @@ -153,24 +162,16 @@ describe('multilayer error cases', function() { ] }; ServerOptions.afterLayergroupCreateCalls = 0; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(layergroup) - }, {}, function(res) { - try { - assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); - // See http://github.com/CartoDB/Windshaft/issues/159 - assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); - var parsed = JSON.parse(res.body); - assert.ok(parsed); - assert.equal(parsed.errors.length, 1); - var error = parsed.errors[0]; - assert.ok(error.match(/column "missing" does not exist/m), error); - // TODO: check which layer introduced the problem ? - done(); - } catch (err) { done(err); } + this.client = new TestClient(layergroup); + this.client.getLayergroup({status: 400}, function(err, parsed) { + assert.ok(!err, err); + // See http://github.com/CartoDB/Windshaft/issues/159 + assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); + assert.ok(parsed); + assert.equal(parsed.errors.length, 1); + var error = parsed.errors[0]; + assert.ok(error.match(/column "missing" does not exist/m), error); + done(); }); }); diff --git a/test/acceptance/ported/server_png8_format.js b/test/acceptance/ported/server_png8_format.js index 30b5f2bc..a710cbd0 100644 --- a/test/acceptance/ported/server_png8_format.js +++ b/test/acceptance/ported/server_png8_format.js @@ -16,13 +16,13 @@ describe('server_png8_format', function() { var serverOptionsPng32 = ServerOptions; serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore); serverOptionsPng32.grainstore.mapnik_tile_format = 'png32'; - var serverPng32 = new cartodbServer(serverOptionsPng32); + var serverPng32 = cartodbServer(serverOptionsPng32); serverPng32.setMaxListeners(0); var serverOptionsPng8 = ServerOptions; serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore); serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h'; - var serverPng8 = new cartodbServer(serverOptionsPng8); + var serverPng8 = cartodbServer(serverOptionsPng8); serverPng8.setMaxListeners(0); diff --git a/test/acceptance/special-numeric-values.js b/test/acceptance/special-numeric-values.js new file mode 100644 index 00000000..2b596b8e --- /dev/null +++ b/test/acceptance/special-numeric-values.js @@ -0,0 +1,71 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); + +describe('special numeric values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var ATTRIBUTES_LAYER = 1; + + function createMapConfig(sql, id, columns) { + return { + version: '1.6.0', + layers: [ + { + type: 'mapnik', + options: { + sql: "select 1 as id, 'SRID=4326;POINT(0 0)'::geometry as the_geom", + cartocss: '#style { }', + cartocss_version: '2.0.1' + } + }, + { + type: 'mapnik', + options: { + sql: sql || "select 1 as i, 6 as n, 'SRID=4326;POINT(0 0)'::geometry as the_geom", + attributes: { + id: id || 'i', + columns: columns || ['n'] + }, + cartocss: '#style { }', + cartocss_version: '2.0.1' + } + } + ] + }; + } + + it('should retrieve special numeric values', function (done) { + var featureId = 1; + var sql = [ + 'SELECT', + ' 1 as cartodb_id,', + ' null::geometry the_geom_webmercator,', + ' \'infinity\'::float as infinity,', + ' \'-infinity\'::float as _infinity,', + ' \'NaN\'::float as nan' + ].join('\n'); + var id = 'cartodb_id'; + var columns = ['infinity', '_infinity', 'nan']; + + var mapConfig = createMapConfig(sql, id, columns); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getFeatureAttributes(featureId, ATTRIBUTES_LAYER, {}, function (err, attributes) { + assert.ifError(err); + assert.equal(attributes.infinity, 'Infinity'); + assert.equal(attributes._infinity, '-Infinity'); + assert.equal(attributes.nan, 'NaN'); + done(); + }); + }); +}); + diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js new file mode 100644 index 00000000..cdac7bb1 --- /dev/null +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -0,0 +1,279 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +describe('Create mapnik layergroup', function() { + before(function() { + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; + global.environment.enabledFeatures.layerStats = true; + }); + + after(function() { + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; + }); + + var cartocssVersion = '2.3.0'; + var cartocss = '#layer { line-width:16; }'; + + var mapnikLayer1 = { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 1', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer2 = { + type: 'mapnik', + options: { + sql: 'select * from test_table_2 limit 2', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer3 = { + type: 'mapnik', + options: { + sql: 'select * from test_table_3 limit 3', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer4 = { + type: 'mapnik', + options: { + sql: [ + 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address', + ' from test_table t1, test_table_2 t2', + ' where t1.cartodb_id = t2.cartodb_id;' + ].join(''), + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var httpLayer = { + type: 'http', + options: { + urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png', + subdomains: ['a','b','c'] + } + }; + + var mapnikLayerGeomColumn = { + type: 'mapnik', + options: { + sql: 'select *, the_geom as my_geom from test_table_3 limit 2', + geom_column: 'my_geom', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + function mapnikBasicLayerId(index) { + return 'layer' + index; + } + function typeLayerId(type, index) { + return type + '-' + mapnikBasicLayerId(index); + } + + it('with one mapnik layer should response with meta-stats for that layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); + testClient.drain(done); + }); + }); + + it('with two mapnik layer should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + mapnikLayer2 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); + testClient.drain(done); + }); + }); + + it('with three mapnik layer should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + mapnikLayer2, + mapnikLayer3 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); + assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3); + testClient.drain(done); + }); + }); + + it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer4, + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer3, + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); + assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.ok(!layergroup.metadata.layers[2]); + testClient.drain(done); + }); + }); + + it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + httpLayer + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); + assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[1].type, 'http'); + testClient.drain(done); + }); + }); + + it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + httpLayer, + mapnikLayer1 + ] + }); + + testClient.getLayergroup(function (err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[0].type, 'http'); + assert.ok(!layergroup.metadata.layers[0].meta.cartocss); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); + assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); + testClient.drain(done); + }); + }); + + it('should work with different geom_column', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayerGeomColumn + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + // we don't care about stats here as is an aliased column + assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); + testClient.drain(done); + }); + }); + + it('should not include the stats part if the FF is disabled', function(done) { + global.environment.enabledFeatures.layerStats = false; + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + httpLayer, + mapnikLayer1, + httpLayer + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[0].type, 'http'); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); + assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats')); + assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1)); + assert.equal(layergroup.metadata.layers[2].type, 'http'); + testClient.drain(done); + }); + }); +}); diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js new file mode 100644 index 00000000..ced63b0e --- /dev/null +++ b/test/acceptance/stats/multilayer_stats.js @@ -0,0 +1,225 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +describe('multilayer stats disabled', function() { + + before(function () { + this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; + global.environment.enabledFeatures.layerMetadata = true; + global.environment.enabledFeatures.layerStats = false; + }); + + after(function () { + global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; + }); + + + function testLayerMetadataStats(testScenario) { + + it(testScenario.desc, function(done) { + var mapConfig = { + version: '1.3.0', + layers: testScenario.layers + }; + + var testClient = new TestClient(mapConfig); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + layergroup.metadata.layers.forEach(function (layer) { + if (layer.type !== 'torque' && layer.type !== 'mapnik') { + assert.ok(!('stats' in layer.meta)); + } else if (layer.type !== 'torque') { + assert.ok(!('stats' in layer.meta)); + assert.ok('cartocss' in layer.meta); + } else { + assert.ok('cartocss' in layer.meta); + // check torque metadata at least match in number + var torqueLayers = mapConfig.layers.filter(function(layer) { return layer.type === 'torque'; }); + if (torqueLayers.length) { + assert.equal(Object.keys(layergroup.metadata.torque).length, torqueLayers.length); + } + } + }); + + testClient.drain(done); + }); + }); + } + + var cartocssVersion = '2.3.0'; + var cartocss = '#layer { line-width:16; }'; + var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " + + "st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " + + "st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + + var httpLayer = { + type: 'http', + options: { + urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png', + subdomains: ['a','b','c'] + } + }; + + var torqueLayer = { + type: 'torque', + options: { + sql: "select 1 id, '1970-01-02'::date d, 'POINT(0 0)'::geometry the_geom_webmercator", + cartocss: [ + "Map {", + "-torque-frame-count:2;", + "-torque-resolution:3;", + "-torque-time-attribute:d;", + "-torque-aggregation-function:'count(id)';", + "}" + ].join(' '), + cartocss_version: '2.0.1' + } + }; + + var mapnikLayer = { + type: 'mapnik', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikInteractivityLayer = { + type: 'mapnik', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var cartodbLayer = { + type: 'cartodb', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var cartodbInteractivityLayer = { + type: 'cartodb', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var cartodbWadusInteractivityLayer = { + type: 'cartodb', + options: { + sql: sqlWadus, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'wadus' + } + }; + + var noTypeLayer = { + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var noTypeInteractivityLayer = { + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var testScenarios = [ + { + desc: 'one layer, no interactivity', + layers: [cartodbLayer] + }, + { + desc: 'two layers, different interactivity columns', + layers: [ + cartodbWadusInteractivityLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'torque + interactivity layers', + layers: [ + torqueLayer, + cartodbWadusInteractivityLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'interactivity + torque + interactivity', + layers: [ + cartodbInteractivityLayer, + torqueLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'http + interactivity + torque + no interactivity + torque + interactivity', + layers: [ + httpLayer, + cartodbInteractivityLayer, + torqueLayer, + cartodbLayer, + torqueLayer, + cartodbWadusInteractivityLayer + ] + }, + { + desc: 'mapnik type – two layers, interactivity mix', + layers: [ + mapnikLayer, + mapnikInteractivityLayer + ] + }, + { + desc: 'mapnik type – http + interactivity + torque + interactivity', + layers: [ + httpLayer, + mapnikInteractivityLayer, + torqueLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'no type – two layers, interactivity mix', + layers: [ + noTypeLayer, + noTypeInteractivityLayer + ] + }, + { + desc: 'no type – http + interactivity + torque + interactivity', + layers: [ + httpLayer, + noTypeInteractivityLayer, + torqueLayer, + noTypeInteractivityLayer + ] + } + ]; + + testScenarios.forEach(testLayerMetadataStats); + +}); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 7201b7fb..74d34d82 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -1052,8 +1052,9 @@ describe('template_api', function() { 'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); var cc = res.headers['x-cache-channel']; + var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1'; assert.ok(cc); - assert.ok(cc.match, /ciao/, cc); + assert.equal(cc, expectedCC); // hack simulating restart... server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache var get_request = { @@ -1072,8 +1073,9 @@ describe('template_api', function() { 'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); var cc = res.headers['x-cache-channel']; + var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1'; assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart"); - assert.ok(cc.match, /ciao/, cc); + assert.equal(cc, expectedCC); return null; }, function deleteTemplate(err) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js new file mode 100644 index 00000000..42c468fd --- /dev/null +++ b/test/acceptance/user-database-timeout-limit.js @@ -0,0 +1,786 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); + +const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; + +const pointSleepSql = ` + SELECT + pg_sleep(0.3), + 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const validationPointSleepSql = ` + SELECT + pg_sleep(0.3), + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = pointSleepSql, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id', + countBy = 'cartodb_id', + attributes +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version, + attributes, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); + +const DATASOURCE_TIMEOUT_ERROR = { + errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details' + }] +}; + +describe('user database timeout limit', function () { + describe('dataview', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but dataview request fails due to statement timeout', function (done) { + const params = { + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getDataview('count', params, (err, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + + describe('raster', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching raster tiles', function () { + describe('with user\'s timeout of 200 ms', function () { + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.setUserDatabaseTimeoutLimit(200, (err) => { + if (err) { + return done(err); + } + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ] + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png' + }; + + this.testClient.getStaticCenter(params, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + }); + + describe('with onTileErrorStrategy DISABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.setUserDatabaseTimeoutLimit(200, (err) => { + if (err) { + return done(err); + } + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + + this.testClient.drain(done); + }); + }); + + it('"png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + + it('"static png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + }); + + describe('vector', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching vector tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, done); + }); + + it('"mvt" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'mvt', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + + + describe('interactivity', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details', + layer: { id: 'layer0', index: 0, type: 'mapnik' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching interactivity tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ interactivity: 'val' }); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, done); + }); + + it('"grid.json" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'grid.json', + layers: 'mapnik', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + + describe('torque', function () { + describe('while validating in layergroup creation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details', + layer: { id: 'torque-layer0', index: 0, type: 'torque' } + }] + }); + + done(); + }); + }); + }); + + describe('fetching torque tiles', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + type: 'torque', + cartocss: TestClient.CARTOCSS.TORQUE + }); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, done); + }); + + it('"torque.json" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'torque.json', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + + it('".png" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'torque.png', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => { + assert.ifError(err); + + assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); + + describe('attributes:', function () { + describe('while validating in map instatiation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + attributes: { + id: 'cartodb_id', + columns: [ 'val' ] + } + }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.deepEqual(timeoutError, { + errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details', + layer: { + id: 'layer0', + index: 0, + type: 'mapnik' + } + }] + }); + + done(); + }); + }); + }); + + describe('fetching by feature id', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ + attributes: { + id: 'cartodb_id', + columns: [ 'val' ] + } + }); + + this.testClient = new TestClient(mapconfig, 1234); + + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, done); + }); + + it('fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + featureId: 1, + layer: 0, + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getAttributes(params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); + + done(); + }); + }); + }); + }); + }); +}); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js new file mode 100644 index 00000000..45e41f89 --- /dev/null +++ b/test/acceptance/user-render-timeout-limit.js @@ -0,0 +1,394 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); + +const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; + +const pointSleepSql = ` + SELECT + pg_sleep(0.5), + 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +// during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout +const validationPointSleepSql = ` + SELECT + pg_sleep(0.5), + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = pointSleepSql, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); + +describe('user render timeout limit', function () { + describe('map instantiation => validation', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig({ sql: validationPointSleepSql }); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation fails due to statement timeout', function (done) { + const expectedResponse = { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, { + errors: ["You are over platform\'s limits. Please contact us to know more details"], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: "You are over platform\'s limits. Please contact us to know more details", + layer: { + id: "layer0", + index: 0, + type: "mapnik" + } + }] + }); + + done(); + }); + }); + }); + + describe('raster', function () { + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but tile request fails due to render timeout', function (done) { + this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + }); + + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works and render tile fails', function (done) { + var params = { + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + assert.ifError(err); + + assert.deepEqual(timeoutError, { + errors: ["You are over platform\'s limits. Please contact us to know more details"], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: "You are over platform\'s limits. Please contact us to know more details" + }] + }); + + done(); + }); + }); + }); + }); + + describe('vector', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: 'You are over platform\'s limits. Please contact us to know more details' + }] + }); + + done(); + }); + }); + }); + + describe('interativity', function () { + beforeEach(function (done) { + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) { + const params = { + layers: 'mapnik', + format: 'grid.json', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: 'You are over platform\'s limits. Please contact us to know more details' + }] + }); + + done(); + }); + }); + }); + + describe('static images', function () { + describe('with onTileErrorStrategy ENABLED', function () { + let onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = true; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but static image fails due to render timeout', function (done) { + const params = { + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png' + }; + + this.testClient.getStaticCenter(params, function (err, res, tile) { + assert.ifError(err); + + assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { + assert.ifError(err); + done(); + }); + }); + }); + }); + + describe('with onTileErrorStrategy DISABLED', function() { + var onTileErrorStrategy; + + beforeEach(function (done) { + onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy; + global.environment.enabledFeatures.onTileErrorStrategy = false; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy; + + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works and render tile fails', function (done) { + const params = { + zoom: 0, + lat: 0, + lng: 0, + width: 256, + height: 256, + format: 'png', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getStaticCenter(params, function (err, res, timeoutError) { + assert.ifError(err); + + assert.deepEqual(timeoutError, { + errors: ["You are over platform\'s limits. Please contact us to know more details"], + errors_with_context: [{ + type: 'limit', + subtype: 'render', + message: "You are over platform\'s limits. Please contact us to know more details" + }] + }); + + done(); + }); + }); + }); + }); +}); + diff --git a/test/acceptance/widgets/ported/aggregation.js b/test/acceptance/widgets/ported/aggregation.js index 9329f039..62c998a5 100644 --- a/test/acceptance/widgets/ported/aggregation.js +++ b/test/acceptance/widgets/ported/aggregation.js @@ -322,6 +322,25 @@ describe('widgets', function() { }); }); }); + + [adm0name].forEach(function(userQuery) { + it('should search with sum aggregation: ' + userQuery, function(done) { + this.testClient = new TestClient(aggregationSumMapConfig); + this.testClient.widgetSearch('adm0name', userQuery, function (err, res, searchResult) { + assert.ok(!err, err); + assert.ok(searchResult); + assert.equal(searchResult.type, 'aggregation'); + + assert.equal(searchResult.categories.length, 1); + assert.deepEqual( + searchResult.categories, + [{ category:"Argentina", value:28015640 }] + ); + + done(); + }); + }); + }); }); }); diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 1d006ac4..74f4544c 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -218,6 +218,114 @@ describe('widgets-regressions', function() { }); }); + + it('should not count the polygons outside the bounding box', function(done) { + + // $ % $ = not intersecting left triangle + // $$ **VVVVV** %% % = not intersecting right triangle + // $$$ *VVVVV* %%% * = intersecting triangle + // $$$$ ***** %%%% V = bounding box + // $$$$$ *** %%%%% + // $$$$$$ * %%%%%% + // $$$$$$$ %%%%%%% + // $$$$$$$$ %%%%%%%% + + const notIntersectingLeftTriangle = { + type: "Polygon", + coordinates:[[ + [-161.015625,69.28725695167886], + [-162.7734375,-7.710991655433217], + [-40.78125,-8.059229627200192], + [-161.015625,69.28725695167886] + ]] + }; + + const notIntersectingRightTriangle = { + type: "Polygon", + coordinates: [[ + [-29.179687499999996,-7.01366792756663], + [103.71093749999999,-6.664607562172573], + [105.46875,69.16255790810501], + [-29.179687499999996,-7.01366792756663] + ]] + }; + + const intersectingTriangle = { + type: "Polygon", + coordinates:[[ + [-117.42187500000001,68.13885164925573], + [-35.859375,20.96143961409684], + [59.4140625,68.52823492039876], + [-117.42187500000001,68.13885164925573] + ]] + }; + + const query = ` + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(notIntersectingLeftTriangle)}' + ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id, 'notIntersectingLeftTriangle' AS name + UNION + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(notIntersectingRightTriangle)}' + ), 4326), 3857), 2, 'notIntersectingRightTriangle' + UNION + SELECT + ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( + '${JSON.stringify(intersectingTriangle)}' + ), 4326), 3857), 3, 'intersectingTriangle' + `; + + const mapConfig = { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + dataviews: { + val_formula: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: "name", + aggregation: "count", + } + } + }, + analyses: [ + { + "id": "a0", + "type": "source", + "params": { + "query": query + } + } + ] + }; + + this.testClient = new TestClient(mapConfig, 1234); + const params = { + bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658' + }; + this.testClient.getDataview('val_formula', params, function(err, dataview) { + assert.ifError(err); + assert.equal(dataview.categories.length, 1); + assert.equal(dataview.categories[0].category, 'intersectingTriangle'); + done(); + }); + }); + }); }); diff --git a/test/acceptance/x_cache_channel.js b/test/acceptance/x_cache_channel.js deleted file mode 100644 index 4ebb45e8..00000000 --- a/test/acceptance/x_cache_channel.js +++ /dev/null @@ -1,307 +0,0 @@ -var testHelper = require('../support/test_helper'); - -var assert = require('../support/assert'); -var qs = require('querystring'); - -var CartodbWindshaft = require('../../lib/cartodb/server'); -var serverOptions = require('../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); -server.setMaxListeners(0); - -var LayergroupToken = require('../support/layergroup-token'); - -describe('get requests x-cache-channel', function() { - - var keysToDelete; - beforeEach(function() { - keysToDelete = {}; - }); - - afterEach(function(done) { - testHelper.deleteRedisKeys(keysToDelete, done); - }); - - var statusOkResponse = { - status: 200 - }; - - var mapConfig = { - version: '1.3.0', - layers: [ - { - options: { - sql: 'select * from test_table limit 2', - cartocss: '#layer { marker-fill:red; }', - cartocss_version: '2.3.0', - attributes: { - id:'cartodb_id', - columns: [ - 'name', - 'address' - ] - } - } - } - ] - }; - - var layergroupRequest = { - url: '/api/v1/map?config=' + encodeURIComponent(JSON.stringify(mapConfig)), - method: 'GET', - headers: { - host: 'localhost' - } - }; - - function getRequest(url, addApiKey, callbackName) { - var params = {}; - if (!!addApiKey) { - params.api_key = '1234'; - } - if (!!callbackName) { - params.callback = callbackName; - } - - return { - url: url + '?' + qs.stringify(params), - method: 'GET', - headers: { - host: 'localhost', - 'Content-Type': 'application/json' - } - }; - } - - function validateXCacheChannel(done, expectedCacheChannel) { - return function(res, err) { - if (err) { - return done(err); - } - - assert.ok(res.headers['x-cache-channel']); - if (expectedCacheChannel) { - assert.equal(res.headers['x-cache-channel'], expectedCacheChannel); - } - - done(); - }; - } - - function noXCacheChannelHeader(done) { - return function(res, err) { - if (err) { - return done(err); - } - - assert.ok( - !res.headers['x-cache-channel'], - 'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`' - ); - done(); - }; - } - - function withLayergroupId(callback) { - assert.response( - server, - layergroupRequest, - statusOkResponse, - function(res, err) { - if (err) { - return callback(err); - } - var layergroupId = JSON.parse(res.body).layergroupid; - keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - callback(null, layergroupId, res); - } - ); - } - - describe('header should be present', function() { - - it('/api/v1/map Map instantiation', function(done) { - var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table'); - withLayergroupId(function(err, layergroupId, res) { - testFn(res); - }); - }); - - it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) { - withLayergroupId(function(err, layergroupId) { - assert.response( - server, - getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'), - validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table') - ); - }); - }); - - }); - - describe('header should NOT be present', function() { - - it('/', function(done) { - assert.response( - server, - getRequest('/'), - statusOkResponse, - noXCacheChannelHeader(done) - ); - }); - - it('/version', function(done) { - assert.response( - server, - getRequest('/version'), - statusOkResponse, - noXCacheChannelHeader(done) - ); - }); - - it('/health', function(done) { - assert.response( - server, - getRequest('/health'), - statusOkResponse, - noXCacheChannelHeader(done) - ); - }); - - it('/api/v1/map/named list named maps', function(done) { - assert.response( - server, - getRequest('/api/v1/map/named', true), - statusOkResponse, - noXCacheChannelHeader(done) - ); - }); - - describe('with named maps', function() { - - var templateName = 'x_cache'; - - beforeEach(function(done) { - var template = { - version: '0.0.1', - name: templateName, - auth: { - method: 'open' - }, - layergroup: mapConfig - }; - - var namedMapRequest = { - url: '/api/v1/map/named?api_key=1234', - method: 'POST', - headers: { - host: 'localhost', - 'Content-Type': 'application/json' - }, - data: JSON.stringify(template) - }; - - assert.response( - server, - namedMapRequest, - statusOkResponse, - function(res, err) { - done(err); - } - ); - }); - - afterEach(function(done) { - assert.response( - server, - { - url: '/api/v1/map/named/' + templateName + '?api_key=1234', - method: 'DELETE', - headers: { - host: 'localhost' - } - }, - { - status: 204 - }, - function(res, err) { - done(err); - } - ); - }); - - - it('/api/v1/map/named/:template_id Named map retrieval', function(done) { - assert.response( - server, - getRequest('/api/v1/map/named/' + templateName, true), - statusOkResponse, - noXCacheChannelHeader(done) - ); - }); - - it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) { - assert.response( - server, - getRequest('/api/v1/map/named/' + templateName, true, 'cb'), - statusOkResponse, - noXCacheChannelHeader(done) - ); - }); - - }); - - }); - - -}); diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png new file mode 100644 index 00000000..798f7cca Binary files /dev/null and b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png differ diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson new file mode 100644 index 00000000..e89c71dd --- /dev/null +++ b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[-53839,4629161]},"properties":{"name":"Alicante","cartodb_id":1200}},{"type":"Feature","geometry":{"type":"Point","coordinates":[242835,5069332]},"properties":{"name":"Barcelona","cartodb_id":5330}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-5567,4861644]},"properties":{"name":"Castello","cartodb_id":1201}},{"type":"Feature","geometry":{"type":"Point","coordinates":[272735,5092314]},"properties":{"name":"Mataro","cartodb_id":615}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125787,4576600]},"properties":{"name":"Murcia","cartodb_id":952}},{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139148,5030112]},"properties":{"name":"Tarragona","cartodb_id":616}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-44746,4791667]},"properties":{"name":"Valencia","cartodb_id":5942}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-99072,5108695]},"properties":{"name":"Zaragoza","cartodb_id":5932}}]} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json new file mode 100644 index 00000000..c06b8d91 --- /dev/null +++ b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json @@ -0,0 +1 @@ +{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !! ","!!! !!!!! ","!!!!!!! ! ","!!! !!!!! "," !! ! "," "," "," "," "," "," "," "," ### # "," ####### ###"," ####### ## ","$ ## #### ## ","$$ ","$$ ","$$ "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","9","2","1"],"data":{"1":{"cartodb_id":5942},"2":{"cartodb_id":5500},"9":{"cartodb_id":1201}}} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png new file mode 100644 index 00000000..b3088d37 Binary files /dev/null and b/test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png differ diff --git a/test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json b/test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json new file mode 100644 index 00000000..24581bc3 --- /dev/null +++ b/test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json @@ -0,0 +1 @@ +{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !!! ! "," !!!!!!! !!!"," !!!!!!! !! "," !! !!!! !! "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","1"],"data":{"1":{"cartodb_id":5500}}} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson new file mode 100644 index 00000000..500fb3ea --- /dev/null +++ b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}}]} \ No newline at end of file diff --git a/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt new file mode 100644 index 00000000..b87f1b9a Binary files /dev/null and b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt differ diff --git a/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt new file mode 100644 index 00000000..c6e54212 Binary files /dev/null and b/test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt differ diff --git a/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png b/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png new file mode 100644 index 00000000..f7fee54c Binary files /dev/null and b/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png differ diff --git a/test/support/assert.js b/test/support/assert.js index a53d6fbf..c5e22e93 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -126,22 +126,25 @@ assert.response = function(server, req, res, callback) { // Assert response body if (res.body) { var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body; - assert.ok( - eql, - colorize('[red]{Invalid response body.}\n' + + if (!eql) { + return callback(response, new Error(colorize( + '[red]{Invalid response body.}\n' + ' Expected: [green]{' + res.body + '}\n' + - ' Got: [red]{' + response.body + '}') - ); + ' Got: [red]{' + response.body + '}')) + ); + } } // Assert response status if (typeof status === 'number') { - assert.equal(response.statusCode, status, - colorize('[red]{Invalid response status code.}\n' + + if (response.statusCode != status) { + return callback(response, new Error(colorize( + '[red]{Invalid response status code.}\n' + ' Expected: [green]{' + status + '}\n' + ' Got: [red]{' + response.statusCode + '}\n' + - ' Body: ' + response.body) - ); + ' Body: ' + response.body)) + ); + } } // Assert response headers @@ -152,11 +155,13 @@ assert.response = function(server, req, res, callback) { actual = response.headers[name.toLowerCase()], expected = res.headers[name], headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual; - assert.ok(headerEql, - colorize('Invalid response header [bold]{' + name + '}.\n' + + if (!headerEql) { + return callback(response, new Error(colorize( + 'Invalid response header [bold]{' + name + '}.\n' + ' Expected: [green]{' + expected + '}\n' + - ' Got: [red]{' + actual + '}') - ); + ' Got: [red]{' + actual + '}')) + ); + } } } diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 1a6d929c..dcf1cba7 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -75,8 +75,8 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" - LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check' - REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' + REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount' CURL_ARGS="" for i in ${REMOTE_SQL_SCRIPTS} diff --git a/test/support/sql/cdb_invalidate_varnish.sql b/test/support/sql/cdb_invalidate_varnish.sql new file mode 100644 index 00000000..7cd2d8f1 --- /dev/null +++ b/test/support/sql/cdb_invalidate_varnish.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE FUNCTION CDB_Invalidate_Varnish(table_name TEXT) +RETURNS void AS +$$ +BEGIN +END; +$$ LANGUAGE PLPGSQL; \ No newline at end of file diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index eb495028..996c6b3e 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES INSERT INTO _vovw_1_test_table_overviews VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5); +-- table with overviews whit special float values + +CREATE TABLE test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id; + +SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true); + +ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass); + +INSERT INTO test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1), +(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1), +(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1); + +ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id); + +CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom); +CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE TABLE _vovw_1_test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER; + +INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2); -- analysis tables ----------------------------------------------- @@ -649,3 +721,5 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters int END; $$ LANGUAGE plpgsql; GRANT ALL ON FUNCTION cdb_crankshaft.CDB_KMeans(text, integer, integer) TO :TESTUSER; + +ANALYZE; diff --git a/test/support/test-client.js b/test/support/test-client.js index 75fa51e0..21918665 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -3,7 +3,8 @@ var qs = require('querystring'); var step = require('step'); var urlParser = require('url'); - +var PSQL = require('cartodb-psql'); +var _ = require('underscore'); var mapnik = require('windshaft').mapnik; var LayergroupToken = require('./layergroup-token'); @@ -14,16 +15,33 @@ var helper = require('./test_helper'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); serverOptions.analysis.batch.inlineExecution = true; -var server = new CartodbWindshaft(serverOptions); -function TestClient(mapConfig, apiKey) { - this.mapConfig = mapConfig; +const MAPNIK_SUPPORTED_FORMATS = { + 'png': true, + 'png32': true, + 'grid.json': true, + 'geojson': true, + 'mvt': true +} + +function TestClient(config, apiKey) { + this.mapConfig = isMapConfig(config) ? config : null; + this.template = isTemplate(config) ? config : null; this.apiKey = apiKey; this.keysToDelete = {}; + this.server = new CartodbWindshaft(serverOptions); } module.exports = TestClient; +function isMapConfig(config) { + return config && config.layers; +} + +function isTemplate(config) { + return config && config.layergroup; +} + module.exports.RESPONSE = { ERROR: { status: 400, @@ -63,9 +81,42 @@ module.exports.CARTOCSS = { ' line-width: 0.5;', ' line-opacity: 1;', '}' + ].join('\n'), + + TORQUE: [ + 'Map {', + ' -torque-frame-count: 256;', + ' -torque-animation-duration: 30;', + ' -torque-time-attribute: "cartodb_id";', + ' -torque-aggregation-function: "count(1)";', + ' -torque-resolution: 4;', + ' -torque-data-aggregation: linear;', + '}', + '#layer {', + ' marker-width: 7;', + ' marker-fill: #FFB927;', + ' marker-fill-opacity: 0.9;', + ' marker-line-width: 1;', + ' marker-line-color: #FFF;', + ' marker-line-opacity: 1;', + ' comp-op: lighter;', + '}', + '#layer[frame-offset=1] {', + ' marker-width: 9;', + ' marker-fill-opacity: 0.45;', + '}', + '#layer[frame-offset=2] {', + ' marker-width: 11;', + ' marker-fill-opacity: 0.225;', + '}' ].join('\n') }; +module.exports.SQL = { + EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator', + ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' +} + TestClient.prototype.getWidget = function(widgetName, params, callback) { var self = this; @@ -83,7 +134,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -142,7 +193,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -194,7 +245,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -251,7 +302,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call } url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -318,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -360,7 +411,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } @@ -371,7 +422,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { } url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams); - assert.response(server, + assert.response(self.server, { url: url, method: 'GET', @@ -390,9 +441,115 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }, function finish(err, dataview) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - return callback(err, dataview); + if (err) { + return callback(err); + } + + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(null, dataview); + } + ); +}; + +TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, callback) { + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + var extraParams = {}; + if (this.apiKey) { + extraParams.api_key = this.apiKey; + } + if (params && params.filters) { + extraParams.filters = JSON.stringify(params.filters); + } + + var url = '/api/v1/map'; + if (Object.keys(extraParams).length > 0) { + url += '?' + qs.stringify(extraParams); + } + + var expectedResponse = params.response || { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + var layergroupId; + step( + function createLayergroup() { + var next = this; + assert.response(self.server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + + var parsedBody = JSON.parse(res.body); + + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return next(null, parsedBody.layergroupid); + } + ); + }, + function getFeatureAttributes(err, layergroupId) { + assert.ifError(err); + + var next = this; + + url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId; + + assert.response(self.server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }, + expectedResponse, + function(res, err) { + if (err) { + return next(err); + } + + next(null, JSON.parse(res.body)); + } + ); + }, + function finish(err, attributes) { + if (err) { + return callback(err); + } + + return callback(null, attributes); } ); }; @@ -406,24 +563,77 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } var url = '/api/v1/map'; + var urlNamed = url + '/named'; if (this.apiKey) { url += '?' + qs.stringify({api_key: this.apiKey}); } var layergroupId; + + if (params.layergroupid) { + layergroupId = params.layergroupid + } + step( - function createLayergroup() { + function createTemplate () { var next = this; - assert.response(server, + + if (!self.template) { + return next(); + } + + if (!self.apiKey) { + return next(new Error('apiKey param is mandatory to create a new template')); + } + + params.placeholders = params.placeholders || {}; + + assert.response(self.server, { - url: url, + url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), method: 'POST', headers: { host: 'localhost', 'Content-Type': 'application/json' }, - data: JSON.stringify(self.mapConfig) + data: JSON.stringify(self.template) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function (res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).template_id); + } + ); + }, + function createLayergroup(err, templateId) { + var next = this; + + if (layergroupId) { + return next(null, layergroupId); + } + + var data = templateId ? params.placeholders : self.mapConfig + var path = templateId ? + urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : + url; + + assert.response(self.server, + { + url: path, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(data) }, { status: 200, @@ -456,6 +666,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { var format = params.format || 'png'; + if (layers === undefined && !MAPNIK_SUPPORTED_FORMATS[format]) { + throw new Error(`Missing layer filter while fetching ${format} tile, review params argument`); + } + url += [z,x,y].join('/'); url += '.' + format; @@ -471,37 +685,76 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }; - var expectedResponse = { + var expectedResponse = Object.assign({}, { status: 200, headers: { - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'image/png' } - }; + }, params.response); + var isPng = format.match(/png$/); if (isPng) { request.encoding = 'binary'; - expectedResponse.headers['Content-Type'] = 'image/png'; } - assert.response(server, request, expectedResponse, function(res, err) { + var isMvt = format.match(/mvt$/); + + if (isMvt) { + request.encoding = 'binary'; + if (expectedResponse.status === 200) { + expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; + } + } + + + var isGeojson = format.match(/geojson$/); + + if (isGeojson) { + request.encoding = 'utf-8'; + expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + + var isGridJSON = format.match(/grid.json$/); + + if (isGridJSON) { + request.encoding = 'utf-8'; + expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + + if (params.contentType) { + expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + + assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); - var obj; - - if (isPng) { - obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - } else { - obj = JSON.parse(res.body); + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/x-protobuf': + body = new mapnik.VectorTile(z, x, y); + body.setDataSync(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; } - next(null, res, obj); + next(null, res, body); }); }, function finish(err, res, image) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } return callback(err, res, image); } ); @@ -526,7 +779,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { url += '?' + qs.stringify({api_key: this.apiKey}); } - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -538,18 +791,124 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { }, expectedResponse, function(res, err) { + // If there is a response, we are still interested in catching the created keys + // to be able to delete them on the .drain() method. + if (res) { + var parsedBody = JSON.parse(res.body); + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + } if (err) { return callback(err); } - var parsedBody = JSON.parse(res.body); + return callback(null, parsedBody); + } + ); +}; - if (parsedBody.layergroupid) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; +TestClient.prototype.getStaticCenter = function (params, callback) { + var self = this; + + let { layergroupid, z, lat, lng, width, height, format } = params + + var url = `/api/v1/map/`; + + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + step( + function createLayergroup() { + var next = this; + + if (layergroupid) { + return next(null, layergroupid); } - return callback(null, parsedBody); + var data = self.mapConfig + var path = url; + + assert.response(self.server, + { + url: path, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(data) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).layergroupid); + } + ); + }, + function getStaticResult(err, _layergroupid) { + assert.ifError(err); + + var next = this; + + layergroupid = _layergroupid; + + url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}` + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + encoding: 'binary', + method: 'GET', + headers: { + host: 'localhost' + } + }; + + var expectedResponse = Object.assign({}, { + status: 200, + headers: { + 'Content-Type': 'image/png' + } + }, params.response); + + assert.response(self.server, request, expectedResponse, function(res, err) { + assert.ifError(err); + + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; + } + + next(null, res, body); + }); + }, + function finish(err, res, image) { + if (layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + return callback(err, res, image); } ); }; @@ -568,7 +927,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { step( function createLayergroup() { var next = this; - assert.response(server, + assert.response(self.server, { url: url, method: 'POST', @@ -629,7 +988,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { } }; - assert.response(server, request, expectedResponse, function(res, err) { + assert.response(self.server, request, expectedResponse, function(res, err) { assert.ifError(err); next(null, res, JSON.parse(res.body)); }); @@ -642,11 +1001,119 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { ); }; +TestClient.prototype.getAttributes = function(params, callback) { + var self = this; + + if (!Number.isFinite(params.featureId)) { + throw new Error('featureId param must be a number') + } + + if (!Number.isFinite(params.layer)) { + throw new Error('layer param must be a number') + } + + var url = '/api/v1/map'; + + if (this.apiKey) { + url += '?' + qs.stringify({ api_key: this.apiKey }); + } + + var layergroupid; + + if (params.layergroupid) { + layergroupid = params.layergroupid + } + + step( + function createLayergroup() { + var next = this; + + if (layergroupid) { + return next(null, layergroupid); + } + + assert.response(self.server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + var parsedBody = JSON.parse(res.body); + + return next(null, parsedBody.layergroupid); + } + ); + }, + function getAttributes(err, _layergroupid) { + assert.ifError(err); + + var next = this; + + layergroupid = _layergroupid; + + url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`; + + if (self.apiKey) { + url += '?' + qs.stringify({api_key: self.apiKey}); + } + + var request = { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }; + + var expectedResponse = params.response || { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + assert.response(self.server, request, expectedResponse, function (res, err) { + assert.ifError(err); + + var attributes = JSON.parse(res.body); + + next(null, res, attributes); + }); + }, + function finish(err, res, attributes) { + if (layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(err, res, attributes); + } + ); +}; + TestClient.prototype.drain = function(callback) { helper.deleteRedisKeys(this.keysToDelete, callback); }; module.exports.getStaticMap = function getStaticMap(templateName, params, callback) { + var self = this; + + self.server = new CartodbWindshaft(serverOptions); + if (!callback) { callback = params; params = null; @@ -677,9 +1144,56 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba // this could be removed once named maps are invalidated, otherwise you hits the cache var server = new CartodbWindshaft(serverOptions); - assert.response(server, requestOptions, expectedResponse, function (res, err) { + assert.response(self.server, requestOptions, expectedResponse, function (res, err) { helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() { return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary'))); }); }); }; + +TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) { + const userTimeoutLimitsKey = `limits:timeout:${user}`; + const params = [ + userTimeoutLimitsKey, + 'render', userTimeoutLimit, + 'render_public', userTimeoutLimit + ]; + + this.keysToDelete[userTimeoutLimitsKey] = 5; + + helper.configureMetadata('hmset', params, callback); +}; + +TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { + const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; + const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) + const publicuser = global.environment.postgres.user; + + // we need to guarantee all new connections have the new settings + helper.cleanPGPoolConnections() + + const psql = new PSQL({ + user: 'postgres', + dbname: dbname, + host: global.environment.postgres.host, + port: global.environment.postgres.port + }); + + step( + function configureTimeouts () { + const timeoutSQLs = [ + `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`, + `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}` + ]; + + const group = this.group(); + + timeoutSQLs.forEach(sql => psql.query(sql, group())); + }, + callback + ); +}; + + diff --git a/test/support/test_helper.js b/test/support/test_helper.js index de9a6e3c..49c6c1ee 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -14,6 +14,7 @@ var lzmaWorker = new LZMA(); var redis = require('redis'); var nock = require('nock'); var log4js = require('log4js'); +var pg = require('pg'); // set environment specific variables global.environment = require(__dirname + '/../../config/environments/test'); @@ -127,6 +128,11 @@ afterEach(function(done) { }); }); +function cleanPGPoolConnections () { + // TODO: this method will be replaced by psql.end + pg.end(); +} + function deleteRedisKeys(keysToDelete, callback) { if (Object.keys(keysToDelete).length === 0) { @@ -166,12 +172,30 @@ function rmdirRecursiveSync(dirname) { } } +function configureMetadata(action, params, callback) { + redisClient.SELECT(5, function (err) { + if (err) { + return callback(err); + } + + redisClient[action](params, function (err) { + if (err) { + return callback(err); + } + + return callback(); + }); + }); +} + module.exports = { deleteRedisKeys: deleteRedisKeys, lzma_compress_to_base64: lzma_compress_to_base64, checkNoCache: checkNoCache, checkSurrogateKey: checkSurrogateKey, checkCache: checkCache, - rmdirRecursiveSync: rmdirRecursiveSync + rmdirRecursiveSync: rmdirRecursiveSync, + configureMetadata, + cleanPGPoolConnections }; diff --git a/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..6e47297d --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,153 @@ +var assert = require('assert'); +var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +function getDbConnectionMock () { + return { + query: function(sql, callback) { + return callback(null, { + rows: [{rows: 1}] + }); + } + }; +} + +describe('mapnik-layer-stats', function() { + + beforeEach(function () { + this.dbConnectionMock = getDbConnectionMock(); + this.rendererCacheMock = {}; + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + } + ] + }; + + var testMapConfigTwoLayers = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + ] + }; + + var testMapConfigOneLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + ] + }; + + var testMapConfigTwoLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_3', 'test_table_4'] + } + }, + ] + }; + + it('should return 1 feature for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayers); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); + + it('should return 1 feature for one layers with two tables', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers and two tables', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); +}); diff --git a/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..c9adfad9 --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,36 @@ +var assert = require('assert'); +var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +describe('torque-layer-stats', function () { + + beforeEach(function () { + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'torque', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + } + }, + ] + }; + + it('should return torque stats for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layerId = 0; + var layer = mapConfig.getLayer(layerId); + var testSubject = new TorqueLayerStats(); + testSubject.getStats(layer, {}, function (err, result) { + assert.ifError(err); + assert.deepEqual({}, result); + done(); + }); + }); +}); diff --git a/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js new file mode 100644 index 00000000..7e774023 --- /dev/null +++ b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js @@ -0,0 +1,44 @@ +var PostgresDatasource = require('../../../../lib/cartodb/backends/turbo-carto-postgres-datasource'); +var PSQL = require('cartodb-psql'); +var _ = require('underscore'); +var assert = require('assert'); + +describe('turbo-carto-postgres-datasource', function() { + + beforeEach(function () { + const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; + const psql = new PSQL({ + user: 'postgres', + dbname: dbname, + host: global.environment.postgres.host, + port: global.environment.postgres.port + }); + const sql = [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS values', + 'FROM generate_series(1, 1000) x' + ].join('\n'); + this.datasource = new PostgresDatasource(psql, sql); + }); + + it('should ignore NaNs and Infinities when computing ramps', function(done) { + var column = 'values'; + var buckets = 4; + var method = 'equal'; + this.datasource.getRamp(column, buckets, method, function(err, result) { + var expected_result = { + ramp: [ 252, 501, 750, 999 ], + stats: { min_val: 3, max_val: 999, avg_val: 501 }, + strategy: undefined + }; + assert.deepEqual(result, expected_result); + done(); + }); + }); +}); diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js new file mode 100644 index 00000000..9a41030a --- /dev/null +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -0,0 +1,36 @@ +var assert = require('assert'); +var testHelper = require('../../support/test_helper'); + +var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma'); + +describe('lzma-middleware', function() { + + it('it should extend params with decoded lzma', function(done) { + var qo = { + config: { + version: '1.3.0' + } + }; + testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { + var req = { + headers: { + host:'localhost' + }, + query: { + api_key: 'test', + lzma: data + } + }; + lzmaMiddleware(req, {}, function(err) { + if ( err ) { + return done(err); + } + var query = req.query; + assert.deepEqual(qo.config, query.config); + assert.equal('test', query.api_key); + done(); + }); + }); + }); + +}); diff --git a/test/unit/cartodb/model/filter/bbox-filters.test.js b/test/unit/cartodb/model/filter/bbox-filters.test.js index df894892..68327a7a 100644 --- a/test/unit/cartodb/model/filter/bbox-filters.test.js +++ b/test/unit/cartodb/model/filter/bbox-filters.test.js @@ -111,6 +111,23 @@ describe('Bounding box filter', function() { createRef([-180, -45, 0, 45]) ); }); + + it('generating multiple bbox', function() { + var bbox = [90, -45, 190, 45]; + var bboxFilter = createFilter(bbox); + + assert.equal(bboxFilter.bboxes.length, 2); + + assert.deepEqual( + bboxFilter.bboxes[0], + createRef([90, -45, 180, 45]) + ); + assert.deepEqual( + bboxFilter.bboxes[1], + createRef([-180, -45, -170, 45]) + ); + }); + }); describe('out of bounds', function() { diff --git a/test/unit/cartodb/ported/tile_stats.test.js b/test/unit/cartodb/ported/tile_stats.test.js index e5ad622d..b71f1384 100644 --- a/test/unit/cartodb/ported/tile_stats.test.js +++ b/test/unit/cartodb/ported/tile_stats.test.js @@ -6,10 +6,13 @@ var LayergroupController = require('../../../../lib/cartodb/controllers/layergro describe('tile stats', function() { - after(function() { - global.statsClient = null; + beforeEach(function () { + this.statsClient = global.statsClient; }); + afterEach(function() { + global.statsClient = this.statsClient; + }); it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() { var expectedCalls = 2, // it will call increment once for the general error @@ -26,12 +29,14 @@ describe('tile stats', function() { var layergroupController = new LayergroupController(); var reqMock = { + profiler: { toJSONString:function() {} }, params: { format: invalidFormat } }; var resMock = { status: function() { return this; }, + set: function() {}, json: function() {}, jsonp: function() {}, send: function() {} @@ -54,12 +59,14 @@ describe('tile stats', function() { } }); var reqMock = { + profiler: { toJSONString:function() {} }, params: { format: validFormat } }; var resMock = { status: function() { return this; }, + set: function() {}, json: function() {}, jsonp: function() {}, send: function() {} diff --git a/test/unit/cartodb/req2params.test.js b/test/unit/cartodb/req2params.test.js index 6293199e..7b055e99 100644 --- a/test/unit/cartodb/req2params.test.js +++ b/test/unit/cartodb/req2params.test.js @@ -1,6 +1,6 @@ var assert = require('assert'); var _ = require('underscore'); -var test_helper = require('../../support/test_helper'); +require('../../support/test_helper'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); @@ -98,34 +98,31 @@ describe('req2params', function() { }); }); - it('it should extend params with decoded lzma', function(done) { - var qo = { - config: { - version: '1.3.0' + it('it should remove invalid params', function(done) { + var config = { + version: '1.3.0' + }; + var req = { + headers: { + host:'localhost' + }, + query: { + non_included: 'toberemoved', + api_key: 'test', + style: 'override', + config: config } }; - test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { - var req = { - headers: { - host:'localhost' - }, - query: { - non_included: 'toberemoved', - api_key: 'test', - style: 'override', - lzma: data - } - }; - baseController.req2params(prepareRequest(req), function(err, req) { - if ( err ) { - return done(err); - } - var query = req.params; - assert.deepEqual(qo.config, query.config); - assert.equal('test', query.api_key); - assert.equal(undefined, query.non_included); - done(); - }); + baseController.req2params(prepareRequest(req), function(err, req) { + if (err) { + return done(err); + } + var query = req.params; + assert.deepEqual(config, query.config); + assert.equal('test', query.api_key); + assert.equal(undefined, query.non_included); + assert.equal(undefined, query.style); + done(); }); }); diff --git a/yarn.lock b/yarn.lock index 89e0eb13..17bf4db8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -22,12 +26,21 @@ accepts@~1.2.12: negotiator "0.5.3" ajv@^4.9.1: - version "4.11.5" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.5.tgz#b6ee74657b993a01dce44b7944d56f485828d5bd" + version "4.11.8" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" dependencies: co "^4.6.0" json-stable-stringify "^1.0.1" +ajv@^5.1.0: + version "5.2.2" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.2.tgz#47c68d69e86f5d953103b0074a9430dc63da5e39" + dependencies: + co "^4.6.0" + fast-deep-equal "^1.0.0" + json-schema-traverse "^0.3.0" + json-stable-stringify "^1.0.1" + align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -48,20 +61,16 @@ ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" -ap@~0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" - aproba@^1.0.3: - version "1.1.1" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" + version "1.2.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" are-we-there-yet@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz#80e470e95a084794fe1899262c5667c6e88de1b3" + version "1.1.4" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.4.tgz#bb5dca382bb94f05e15194373d16fd3ba1ca110d" dependencies: delegates "^1.0.0" - readable-stream "^2.0.0 || ^1.1.13" + readable-stream "^2.0.6" argparse@^1.0.7: version "1.0.9" @@ -105,13 +114,17 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws4@^1.2.1: +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + +aws4@^1.2.1, aws4@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -balanced-match@^0.4.1: - version "0.4.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -120,8 +133,8 @@ bcrypt-pbkdf@^1.0.0: tweetnacl "^0.14.3" bindings@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.2.1.tgz#14ad6113812d2d37d72e67b4cacb4bb726505f11" + version "1.3.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7" block-stream@*: version "0.0.9" @@ -150,16 +163,28 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.0.0: - version "1.1.6" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.6.tgz#7197d7eaa9b87e648390ea61fc66c84427420df9" +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" dependencies: - balanced-match "^0.4.1" + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + dependencies: + hoek "4.x.x" + +brace-expansion@^1.1.7: + version "1.1.8" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" + dependencies: + balanced-match "^1.0.0" concat-map "0.0.1" -buffer-shims@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" +browser-stdout@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" buffer-writer@1.0.1: version "1.0.1" @@ -194,18 +219,18 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.50.3: - version "0.50.3" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.50.3.tgz#dc6c5e3b0b39bc970c8ab6e66a61d905954eb85a" +camshaft@0.58.1: + version "0.58.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.1.tgz#e4156580683f624212ea3020e59790ad006f24cc" dependencies: async "^1.5.2" bunyan "1.8.1" - cartodb-psql "0.7.1" + cartodb-psql "^0.10.1" debug "^2.2.0" dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +248,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -245,22 +270,21 @@ cartocolor@4.0.0: dependencies: colorbrewer "1.0.0" -cartodb-psql@0.7.1, cartodb-psql@~0.7.1: - version "0.7.1" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.7.1.tgz#578ce04db9262f1296845dec643461105a594e22" +cartodb-psql@0.10.1, cartodb-psql@^0.10.1: + version "0.10.1" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.1.tgz#0ac947e62fe10b27916df6b7ba6c461953fe3a23" dependencies: debug "~2.2.0" - pg cartodb/node-postgres#6.1.2-cdb1 - step "~0.0.6" + pg cartodb/node-postgres#6.1.6-cdb1 underscore "~1.6.0" cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" -cartodb-redis@0.13.2: - version "0.13.2" - resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.13.2.tgz#de5214fa5c3ab336c4da978133efa8f908b3691c" +cartodb-redis@0.14.0: + version "0.14.0" + resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.14.0.tgz#6f82fdb3e5b7c8005dbaccd6172c1706c4378df2" dependencies: dot "~1.0.2" redis-mpool "~0.4.1" @@ -303,12 +327,12 @@ chroma-js@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.1.1.tgz#9bb9434959336ece75700aaadfeedc71806d8c05" -cli@0.6.x: - version "0.6.6" - resolved "https://registry.yarnpkg.com/cli/-/cli-0.6.6.tgz#02ad44a380abf27adac5e6f0cdd7b043d74c53e3" +cli@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cli/-/cli-1.0.1.tgz#22817534f24bfa4950c34d532d48ecbc621b8c14" dependencies: exit "0.1.2" - glob "~ 3.2.1" + glob "^7.1.1" cliui@^2.1.0: version "2.1.0" @@ -344,20 +368,16 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-0.6.1.tgz#fa68a14f6a945d54dbbe50d8cdb3320e9e3b1a06" - -commander@2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.3.0.tgz#fd430e889832ec353b9acd1de217c11cb3eef873" - -commander@^2.9.0: +commander@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" +commander@^2.9.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -377,8 +397,8 @@ content-disposition@0.5.1: resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.1.tgz#87476c6a67c8daa87e32e87616df883ba7fb071b" content-type@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.2.tgz#b7d113aee7a8dd27bd21133c4dc2529df1721eed" + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" cookie-signature@1.0.6: version "1.0.6" @@ -388,7 +408,7 @@ cookie@0.1.5: version "0.1.5" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.1.5.tgz#6ab9948a4b1ae21952cd2588530a4722d4044d7c" -core-util-is@~1.0.0: +core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -398,6 +418,12 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" +cryptiles@3.x.x: + version "3.1.2" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" + dependencies: + boom "5.x.x" + d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -412,23 +438,29 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.0.0.tgz#89bd9df6732b51256bc6705342bba02ed12131ef" - dependencies: - ms "0.6.2" - -debug@2.2.0, debug@^2.2.0, debug@~2.2.0: +debug@2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: ms "0.7.1" -debug@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" +debug@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b" dependencies: - ms "0.6.2" + ms "0.7.2" + +debug@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" + dependencies: + ms "2.0.0" + +debug@^2.2.0: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + dependencies: + ms "2.0.0" decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" @@ -445,8 +477,8 @@ deep-equal@^1.0.0: resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" deep-extend@~0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.1.tgz#efe4113d08085f4e6f9687759810f807469e2253" + version "0.4.2" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.2.tgz#48b699c27e334bf89f10892be432f6e4c7d34a7f" deep-is@~0.1.3: version "0.1.3" @@ -461,16 +493,16 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" depd@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.0.tgz#e1bd82c6aab6ced965b97b88b17ed3e528ca18c3" + version "1.1.1" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" destroy@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" -diff@1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/diff/-/diff-1.0.8.tgz#343276308ec991b7bc82267ed55bc1411f971666" +diff@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9" dom-serializer@0: version "0.1.0" @@ -500,7 +532,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -542,9 +578,9 @@ escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" -escape-string-regexp@1.0.2, escape-string-regexp@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.2.tgz#4dbc2fe674e71949caf3fb2695ce7f2dc1d9a8d1" +escape-string-regexp@1.0.5, escape-string-regexp@^1.0.2: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" escodegen@1.8.x: version "1.8.1" @@ -561,9 +597,9 @@ esprima@2.7.x, esprima@^2.7.1: version "2.7.3" resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" -esprima@^3.1.1: - version "3.1.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" +esprima@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" estraverse@^1.9.1: version "1.9.3" @@ -611,13 +647,17 @@ express@~4.13.3: utils-merge "1.0.0" vary "~1.0.1" -extend@~3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4" +extend@~3.0.0, extend@~3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" -extsprintf@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.0.2.tgz#e1080e0658e300b06294990cc70e1502235fd550" +extsprintf@1.3.0, extsprintf@^1.2.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + +fast-deep-equal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" fast-levenshtein@~2.0.4: version "2.0.6" @@ -650,16 +690,24 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" form-data@~2.1.1: - version "2.1.2" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.2.tgz#89c3534008b97eada4cbb157d58f6f5df025eae4" + version "2.1.4" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1" + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.5" + mime-types "^2.1.12" + +form-data@~2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" dependencies: asynckit "^0.4.0" combined-stream "^1.0.5" mime-types "^2.1.12" forwarded@~0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.0.tgz#19ef9874c4ae1c297bcf078fde63a09b66a84363" + version "0.1.2" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" fresh@0.3.0: version "0.3.0" @@ -686,9 +734,9 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: mkdirp ">=0.5 0" rimraf "2" -gauge@~2.7.1: - version "2.7.3" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.3.tgz#1c23855f962f17b3ad3d0dc7443f304542edfe09" +gauge@~2.7.3: + version "2.7.4" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" dependencies: aproba "^1.0.3" console-control-strings "^1.0.0" @@ -700,11 +748,11 @@ gauge@~2.7.1: wide-align "^1.1.0" gdal@~0.9.2: - version "0.9.4" - resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.4.tgz#6dad4abb8ffb3e0d51150fb7935ad7c622c81818" + version "0.9.6" + resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.6.tgz#0cf75d830d35847b4274368b10e04a925321a0ba" dependencies: - nan "~2.5.0" - node-pre-gyp "~0.6.27" + nan "~2.6.2" + node-pre-gyp "~0.6.36" generate-function@^2.0.0: version "2.0.0" @@ -716,9 +764,9 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.2.tgz#886bc5bf0beb7db96e81bcbba078818de5a62683" +generic-pool@2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" generic-pool@~2.1.1: version "2.1.1" @@ -737,18 +785,21 @@ get-caller-file@^1.0.1: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" getpass@^0.1.1: - version "0.1.6" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.6.tgz#283ffd9fc1256840875311c1b60e8c40187110e6" + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" dependencies: assert-plus "^1.0.0" -glob@3.2.3, "glob@~ 3.2.1": - version "3.2.3" - resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" +glob@7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: - graceful-fs "~2.0.0" + fs.realpath "^1.0.0" + inflight "^1.0.4" inherits "2" - minimatch "~0.2.11" + minimatch "^3.0.2" + once "^1.3.0" + path-is-absolute "^1.0.0" glob@^5.0.15: version "5.0.15" @@ -770,14 +821,14 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5: - version "7.1.1" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" +glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "^3.0.2" + minimatch "^3.0.4" once "^1.3.0" path-is-absolute "^1.0.0" @@ -785,17 +836,13 @@ graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" -graceful-fs@~2.0.0: - version "2.0.3" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-2.0.3.tgz#7cd2cdb228a4a3f36e95efa6cc142de7d1a136d0" - "graceful-readlink@>= 1.0.0": version "1.0.1" resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" grainstore@~1.6.0: - version "1.6.1" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.1.tgz#8950279ea737eb0ce403a85693642b4bed7f8e48" + version "1.6.3" + resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.3.tgz#6900cc811aadc1ed2c00fcd429c672f8b8e1a5cb" dependencies: carto "0.16.3" debug "~2.2.0" @@ -807,13 +854,13 @@ grainstore@~1.6.0: semver "~5.0.3" underscore "~1.6.0" -growl@1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/growl/-/growl-1.8.1.tgz#4b2dec8d907e93db336624dcec0183502f8c9428" +growl@1.9.2: + version "1.9.2" + resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: - version "4.0.6" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.6.tgz#2ce4484850537f9c97a8026d5399b935c4ed4ed7" + version "4.0.10" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -825,6 +872,10 @@ har-schema@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + har-validator@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d" @@ -841,6 +892,13 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" +har-validator@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" + dependencies: + ajv "^5.1.0" + har-schema "^2.0.0" + has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -855,7 +913,7 @@ has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" -hawk@~3.1.3: +hawk@3.1.3, hawk@~3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" dependencies: @@ -864,6 +922,15 @@ hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" +hawk@~6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -875,9 +942,13 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" +hoek@4.x.x: + version "4.2.0" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" + hosted-git-info@^2.1.4: - version "2.4.1" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.1.tgz#4b0445e41c004a8bd1337773a4ff790ca40318c8" + version "2.5.0" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" htmlparser2@3.8.x: version "3.8.3" @@ -904,6 +975,14 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -919,7 +998,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1: +inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -939,7 +1018,7 @@ is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" -is-buffer@^1.0.2: +is-buffer@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc" @@ -956,8 +1035,8 @@ is-fullwidth-code-point@^1.0.0: number-is-nan "^1.0.0" is-my-json-valid@^2.12.4: - version "2.16.0" - resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.0.tgz#f079dd9bfdae65ee2038aae8acbc86ab109e3693" + version "2.16.1" + resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11" dependencies: generate-function "^2.0.0" generate-object-property "^1.1.0" @@ -984,9 +1063,9 @@ isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" -isexe@^1.1.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/isexe/-/isexe-1.1.2.tgz#36f3e22e60750920f5e7241a476a8c6a42275ad0" +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" isstream@~0.1.2: version "0.1.2" @@ -1011,46 +1090,41 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" -jade@0.26.3: - version "0.26.3" - resolved "https://registry.yarnpkg.com/jade/-/jade-0.26.3.tgz#8f10d7977d8d79f2f6ff862a81b0513ccb25686c" - dependencies: - commander "0.6.1" - mkdirp "0.3.0" - -jodid25519@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" - dependencies: - jsbn "~0.1.0" - js-base64@^2.1.9: - version "2.1.9" - resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" + version "2.3.2" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.3.2.tgz#a79a923666372b580f8e27f51845c6f7e8fbfbaf" + +js-string-escape@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" js-yaml@3.x, js-yaml@^3.4.6: - version "3.8.2" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.2.tgz#02d3e2c0f6beab20248d412c352203827d786721" + version "3.10.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" dependencies: argparse "^1.0.7" - esprima "^3.1.1" + esprima "^4.0.0" jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" -jshint@~2.6.0: - version "2.6.3" - resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.6.3.tgz#84b470b8e5d5cd7adf0a3bd4975250443c9d311a" +jshint@~2.9.4: + version "2.9.5" + resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.5.tgz#1e7252915ce681b40827ee14248c46d34e9aa62c" dependencies: - cli "0.6.x" + cli "~1.0.0" console-browserify "1.1.x" exit "0.1.x" htmlparser2 "3.8.x" - minimatch "1.0.x" + lodash "3.7.x" + minimatch "~3.0.2" shelljs "0.3.x" strip-json-comments "1.0.x" - underscore "1.6.x" + +json-schema-traverse@^0.3.0: + version "0.3.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" json-schema@0.2.3: version "0.2.3" @@ -1066,6 +1140,10 @@ json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" +json3@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/json3/-/json3-3.3.2.tgz#3c0434743df93e2f5c42aee7b19bcb483575f4e1" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -1075,19 +1153,19 @@ jsonpointer@^4.0.0: resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-4.0.1.tgz#4fd92cb34e0e9db3c89c8622ecf51f9b978c6cb9" jsprim@^1.2.2: - version "1.4.0" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.0.tgz#a3b87e40298d8c380552d8cc7628a0bb95a22918" + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" dependencies: assert-plus "1.0.0" - extsprintf "1.0.2" + extsprintf "1.3.0" json-schema "0.2.3" - verror "1.3.6" + verror "1.10.0" kind-of@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.1.0.tgz#475d698a5e49ff5e53d14e3e732429dc8bf4cf47" + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" dependencies: - is-buffer "^1.0.2" + is-buffer "^1.1.5" lazy-cache@^1.0.3: version "1.0.4" @@ -1116,14 +1194,65 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" +lodash._baseassign@^3.0.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz#8c38a099500f215ad09e59f1722fd0c52bfe0a4e" + dependencies: + lodash._basecopy "^3.0.0" + lodash.keys "^3.0.0" + +lodash._basecopy@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz#8da0e6a876cf344c0ad8a54882111dd3c5c7ca36" + +lodash._basecreate@^3.0.0: + version "3.0.3" + resolved "https://registry.yarnpkg.com/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz#1bc661614daa7fc311b7d03bf16806a0213cf821" + +lodash._getnative@^3.0.0: + version "3.9.1" + resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" + +lodash._isiterateecall@^3.0.0: + version "3.0.9" + resolved "https://registry.yarnpkg.com/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz#5203ad7ba425fae842460e696db9cf3e6aac057c" + lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.1.0, lodash.assign@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" +lodash.create@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/lodash.create/-/lodash.create-3.1.1.tgz#d7f2849f0dbda7e04682bb8cd72ab022461debe7" + dependencies: + lodash._baseassign "^3.0.0" + lodash._basecreate "^3.0.0" + lodash._isiterateecall "^3.0.0" + +lodash.isarguments@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + +lodash.isarray@^3.0.0: + version "3.0.4" + resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55" + +lodash.keys@^3.0.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a" + dependencies: + lodash._getnative "^3.0.0" + lodash.isarguments "^3.0.0" + lodash.isarray "^3.0.0" + lodash@2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.1.tgz#5b7723034dda4d262e5a46fb2c58d7cc22f71420" +lodash@3.7.x: + version "3.7.0" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.7.0.tgz#3678bd8ab995057c07ade836ed2ef087da811d45" + lodash@^4.5.1: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -1141,7 +1270,7 @@ longest@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" -lru-cache@2, lru-cache@2.6.5: +lru-cache@2.6.5: version "2.6.5" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.6.5.tgz#e56d6354148ede8d7707b58d143220fd08df0fd5" @@ -1206,17 +1335,17 @@ millstone@0.6.17: underscore "~1.6.0" zipfile "~0.5.5" -mime-db@~1.26.0: - version "1.26.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.26.0.tgz#eaffcd0e4fc6935cf8134da246e2e6c35305adff" +mime-db@~1.30.0: + version "1.30.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" -mime-types@^2.1.12, mime-types@~2.1.13, mime-types@~2.1.6, mime-types@~2.1.7: - version "2.1.14" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee" +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, mime-types@~2.1.7: + version "2.1.17" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: - mime-db "~1.26.0" + mime-db "~1.30.0" -mime@1.3.4, mime@~1.3.4: +mime@1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1224,27 +1353,17 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -minimatch@1.0.x: - version "1.0.0" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" - dependencies: - lru-cache "2" - sigmund "~1.0.0" +mime@~1.3.4: + version "1.3.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: - brace-expansion "^1.0.0" + brace-expansion "^1.1.7" -minimatch@~0.2.11: - version "0.2.14" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.2.14.tgz#c74e780574f63c6f9a090e90efbe6ef53a6a756a" - dependencies: - lru-cache "2" - sigmund "~1.0.0" - -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1252,51 +1371,52 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" -mkdirp@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e" - -mkdirp@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.0.tgz#1d73076a6df986cd9344e15e71fcc05a4c9abf12" - dependencies: - minimist "0.0.8" - -mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: +mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: minimist "0.0.8" -mocha@~1.21.4: - version "1.21.5" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-1.21.5.tgz#7c58b09174df976e434a23b1e8d639873fc529e9" +mocha@~3.4.1: + version "3.4.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" dependencies: - commander "2.3.0" - debug "2.0.0" - diff "1.0.8" - escape-string-regexp "1.0.2" - glob "3.2.3" - growl "1.8.1" - jade "0.26.3" - mkdirp "0.5.0" + browser-stdout "1.3.0" + commander "2.9.0" + debug "2.6.0" + diff "3.2.0" + escape-string-regexp "1.0.5" + glob "7.1.1" + growl "1.9.2" + json3 "3.3.2" + lodash.create "3.1.1" + mkdirp "0.5.1" + supports-color "3.1.2" -moment@^2.10.6: +moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" -ms@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.6.2.tgz#d89c2124c6fdc1353d65a8b77bf1aac4b193708c" - ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" +ms@0.7.2: + version "0.7.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1305,14 +1425,18 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.5.0: - version "2.5.1" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" nan@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" +nan@~2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" + ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" @@ -1332,15 +1456,16 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: - version "0.6.34" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" +node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: + version "0.6.38" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.38.tgz#e92a20f83416415bb4086f6d1fb78b3da73d113d" dependencies: + hawk "3.1.3" mkdirp "^0.5.1" nopt "^4.0.1" npmlog "^4.0.2" rc "^1.1.7" - request "^2.81.0" + request "2.81.0" rimraf "^2.6.1" semver "^5.3.0" tar "^2.2.1" @@ -1364,8 +1489,8 @@ nopt@^4.0.1: osenv "^0.1.4" normalize-package-data@^2.3.2: - version "2.3.6" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.6.tgz#498fa420c96401f787402ba21e600def9f981fff" + version "2.4.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" dependencies: hosted-git-info "^2.1.4" is-builtin-module "^1.0.0" @@ -1373,26 +1498,30 @@ normalize-package-data@^2.3.2: validate-npm-package-license "^3.0.1" npmlog@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.0.2.tgz#d03950e0e78ce1527ba26d2a7592e9348ac3e75f" + version "4.1.2" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" dependencies: are-we-there-yet "~1.1.2" console-control-strings "~1.1.0" - gauge "~2.7.1" + gauge "~2.7.3" set-blocking "~2.0.0" number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1: +oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1459,8 +1588,8 @@ parse-json@^2.2.0: error-ex "^1.2.0" parseurl@~1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56" + version "1.3.2" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" path-exists@^2.0.0: version "2.1.0" @@ -1488,32 +1617,36 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.6.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.6.0.tgz#2e300199927b6d7db6be71e2e3435dddddf07b41" + version "1.8.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" dependencies: - generic-pool "2.4.2" + generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: - version "1.11.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.11.0.tgz#aae91a82d952b633bb88d006350a166daaf6ea90" + version "1.12.1" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2" dependencies: - ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" - postgres-interval "~1.0.0" + postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.2-cdb1: - version "6.1.2" - resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" +"pg@github:cartodb/node-postgres#6.1.6-cdb1": + version "6.1.6" + resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: buffer-writer "1.0.1" + js-string-escape "1.0.1" packet-reader "0.2.0" pg-connection-string "0.1.3" pg-pool "1.*" @@ -1522,8 +1655,8 @@ pg@cartodb/node-postgres#6.1.2-cdb1: semver "4.3.2" pgpass@1.x: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.1.tgz#0de8b5bef993295d90a7e17d976f568dcd25d49f" + version "1.0.2" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" dependencies: split "^1.0.0" @@ -1566,8 +1699,8 @@ postcss@5.0.19: supports-color "^3.1.2" postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8: - version "5.2.16" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.16.tgz#732b3100000f9ff8379a48a53839ed097376ad57" + version "5.2.17" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.17.tgz#cf4f597b864d65c8a492b2eabe9d706c879c388b" dependencies: chalk "^1.1.3" js-base64 "^2.1.9" @@ -1586,9 +1719,9 @@ postgres-date@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" -postgres-interval@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.0.2.tgz#7261438d862b412921c6fdb7617668424b73a6ed" +postgres-interval@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.1.tgz#acdb0f897b4b1c6e496d9d4e0a853e1c428f06f0" dependencies: xtend "^4.0.0" @@ -1643,6 +1776,10 @@ qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" +qs@~6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + queue-async@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" @@ -1660,8 +1797,8 @@ raw-body@~2.1.5: unpipe "1.0.0" rc@^1.1.7: - version "1.1.7" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.1.7.tgz#c5ea564bb07aff9fd3a5b32e906c1d3a65940fea" + version "1.2.1" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.1.tgz#2e03e8e42ee450b8cb3dce65be1bf8974e1dfd95" dependencies: deep-extend "~0.4.0" ini "~1.3.0" @@ -1683,7 +1820,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1692,16 +1829,16 @@ readable-stream@1.1, readable-stream@~1.1.9: isarray "0.0.1" string_decoder "~0.10.x" -"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.1.4: - version "2.2.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.6.tgz#8b43aed76e71483938d12a8d46c6cf1a00b1f816" +readable-stream@^2.0.6, readable-stream@^2.1.4: + version "2.3.3" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" dependencies: - buffer-shims "^1.0.0" core-util-is "~1.0.0" - inherits "~2.0.1" + inherits "~2.0.3" isarray "~1.0.0" process-nextick-args "~1.0.6" - string_decoder "~0.10.x" + safe-buffer "~5.1.1" + string_decoder "~1.0.3" util-deprecate "~1.0.1" readable-stream@~1.0.2: @@ -1713,6 +1850,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1730,32 +1876,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.81.0: +request@2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1782,6 +1903,58 @@ request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@2.x, request@^2.55.0, request@^2.69.0: + version "2.82.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.82.0.tgz#2ba8a92cd7ac45660ea2b10a53ae67cd247516ea" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + hawk "~6.0.2" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + stringstream "~0.0.5" + tough-cookie "~2.3.2" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + +request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1801,8 +1974,8 @@ right-align@^0.1.1: align-text "^0.1.1" rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" + version "2.6.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" dependencies: glob "^7.0.5" @@ -1812,17 +1985,17 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" +safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" semver@4.3.2: version "4.3.2" @@ -1836,6 +2009,10 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + send@0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" @@ -1886,10 +2063,6 @@ shelljs@0.3.x: version "0.3.0" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" -sigmund@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590" - signal-exit@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" @@ -1904,6 +2077,12 @@ sntp@1.x.x: dependencies: hoek "2.x.x" +sntp@2.x.x: + version "2.0.2" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" + dependencies: + hoek "4.x.x" + source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -1911,8 +2090,8 @@ source-map@^0.4.4: amdefine ">=0.0.4" source-map@^0.5.1, source-map@^0.5.6, source-map@~0.5.1: - version "0.5.6" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412" + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" source-map@~0.2.0: version "0.2.0" @@ -1938,13 +2117,17 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/split/-/split-1.0.0.tgz#c4395ce683abcd254bc28fe1dabb6e5c27dcffae" + version "1.0.1" + resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" dependencies: through "2" @@ -1953,11 +2136,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.8" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.8.tgz#4cbcf965d8b901d1b1015cbc7fc415aae157dfaa" + version "3.1.12" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.12.tgz#2b3a14b17162e39e8aa6e1e2487a41d0795396d8" dependencies: - nan "~2.4.0" - node-pre-gyp "~0.6.31" + nan "~2.7.0" + node-pre-gyp "~0.6.38" srs@1.x: version "1.2.0" @@ -1966,8 +2149,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.11.0.tgz#2d8d5ebb4a6fab28ffba37fa62a90f4a3ea59d77" + version "1.13.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -1976,7 +2159,6 @@ sshpk@^1.7.0: optionalDependencies: bcrypt-pbkdf "^1.0.0" ecc-jsbn "~0.1.1" - jodid25519 "^1.0.0" jsbn "~0.1.0" tweetnacl "~0.14.0" @@ -2014,7 +2196,13 @@ string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" -stringstream@~0.0.4: +string_decoder@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" + dependencies: + safe-buffer "~5.1.0" + +stringstream@~0.0.4, stringstream@~0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2038,6 +2226,12 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" +supports-color@3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5" + dependencies: + has-flag "^1.0.0" + supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" @@ -2080,17 +2274,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: - version "2.3.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": + version "2.3.1-cdb4" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb1: - version "0.6.18-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": + version "0.6.18-cdb3" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2113,9 +2307,9 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0: - version "2.3.2" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.2.tgz#f081f76e4c85720e6c37a5faced737150d84072a" +tough-cookie@~2.3.0, tough-cookie@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: punycode "^1.4.1" @@ -2129,9 +2323,9 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@0.19.0: - version "0.19.0" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.0.tgz#83fb1932acd42acb426312eef216b5f6ac34708e" +turbo-carto@0.19.2: + version "0.19.2" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" @@ -2159,19 +2353,20 @@ type-detect@^1.0.0: resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" type-is@~1.6.10, type-is@~1.6.6: - version "1.6.14" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.14.tgz#e219639c17ded1ca0789092dd54a03826b817cb2" + version "1.6.15" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410" dependencies: media-typer "0.3.0" - mime-types "~2.1.13" + mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.14" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.14.tgz#25b15d1af39b21752ee33703adbf432e8bc8f77d" + version "2.8.29" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" dependencies: source-map "~0.5.1" - uglify-to-browserify "~1.0.0" yargs "~3.10.0" + optionalDependencies: + uglify-to-browserify "~1.0.0" uglify-to-browserify@~1.0.0: version "1.0.2" @@ -2181,10 +2376,6 @@ uid-number@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" -underscore@1.6.x, underscore@~1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" - underscore@1.8.2: version "1.8.2" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.2.tgz#64df2eb590899de950782f3735190ba42ebf311d" @@ -2193,6 +2384,10 @@ underscore@1.8.3: version "1.8.3" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" +underscore@~1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" + unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -2205,9 +2400,9 @@ utils-merge@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" -uuid@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" +uuid@^3.0.0, uuid@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" validate-npm-package-license@^3.0.1: version "3.0.1" @@ -2220,27 +2415,29 @@ vary@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/vary/-/vary-1.0.1.tgz#99e4981566a286118dfb2b817357df7993376d10" -verror@1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.3.6.tgz#cff5df12946d297d2baaefaa2689e25be01c005c" +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" dependencies: - extsprintf "1.0.2" + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" which-module@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" which@^1.1.1: - version "1.2.12" - resolved "https://registry.yarnpkg.com/which/-/which-1.2.12.tgz#de67b5e450269f194909ef23ece4ebe416fa1192" + version "1.3.0" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" dependencies: - isexe "^1.1.1" + isexe "^2.0.0" wide-align@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.0.tgz#40edde802a71fea1f070da3e62dcda2e7add96ad" + version "1.1.2" + resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.2.tgz#571e0f1b0604636ebc0dfc21b0339bbe31341710" dependencies: - string-width "^1.0.1" + string-width "^1.0.2" window-size@0.1.0: version "0.1.0" @@ -2250,14 +2447,14 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.0.1.tgz#d06b4673704fe8f8f2e87c1f590c836659ab46f3" +windshaft@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.2.tgz#72efe0dbc0d8d4bcba4211fdabd15dd2e0799df9" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.7.1" + cartodb-psql "^0.10.1" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" @@ -2269,8 +2466,8 @@ windshaft@3.0.1: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb1 - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb1 + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb4 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb3 torque.js "~2.11.0" underscore "~1.6.0"