cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into check-node-version

Browse Source
This commit is contained in:
Raul Ochoa 2017-09-25 14:51:50 +00:00
commit 8daa4bb08a
94 changed files with 8071 additions and 2044 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -8,6 +8,6 @@ tools/munin/windshaft.conf
logs/
pids/
redis.pid
test.log
npm-debug.log
*.log
coverage/
.DS_Store

2
.jshintrc
View File

@ -40,7 +40,7 @@
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
// "eqnull" : false, // true: Tolerate use of `== null`
// "es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
// "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`)
"esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`)
// "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// // (ex: `for each`, multiple try/catch, function expression…)
// "evil" : false, // true: Tolerate use of `eval` and `new Function()`

2
HOWTO_RELEASE
View File

@ -1,7 +1,7 @@
1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json
3. Ensure NEWS section exists for the new version, review it, add release date
4. Recreate yarn.lock with: `yarn upgrade`
4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
5. Commit package.json, yarn.lock, NEWS
6. git tag -a Major.Minor.Patch # use NEWS section as content
7. Stub NEWS/package for next version

279
NEWS.md
View File

@ -1,9 +1,284 @@
# Changelog
## 3.1.2
## 3.12.11
Released 2017-mm-dd
Bugfixes:
- Bounding box parameter ignored in static named maps #735.
## 3.12.10
Released 2017-09-18
- Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2).
## 3.12.9
Released 2017-09-07
Bug fixes:
- Do not use distinct when calculating quantiles. #743
## 3.12.8
Released 2017-09-07
Bug fixes:
- Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962)
## 3.12.7
Released 2017-09-01
- Upgrades camshaft to [0.58.1](https://github.com/CartoDB/camshaft/releases/tag/0.58.1).
## 3.12.6
Released 2017-08-31
- Upgrades camshaft to [0.58.0](https://github.com/CartoDB/camshaft/releases/tag/0.58.0).
## 3.12.5
Released 2017-08-24
- Upgrades camshaft to [0.57.0](https://github.com/CartoDB/camshaft/releases/tag/0.57.0).
## 3.12.4
Released 2017-08-23
Announcements:
- Upgrades camshaft to [0.56.0](https://github.com/CartoDB/camshaft/releases/tag/0.56.0).
## 3.12.3
Released 2017-08-22
Announcements:
- Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8).
## 3.12.2
Released 2017-08-16
Bug fixes:
- Polygon count problems #725.
## 3.12.1
Released 2017-08-13
- Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1).
- Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1).
- Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7).
## 3.12.0
Released 2017-08-10
Announcements:
- Apply max tile response time for requests to layergoup, tiles, static maps, attributes and dataviews services #717.
- Upgrades windshaft to [3.3.0](https://github.com/CartoDB/windshaft/releases/tag/3.3.0).
- Upgrades cartodb-redis to [0.14.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/0.14.0).
## 3.11.0
Released 2017-08-08
Announcements:
- Allow to override with any aggregation for histograms instantiated w/o aggregation.
Bug fixes:
- Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series.
- Support timestamp with timezones to calculate the number of bins in time-series.
- Fixed issue related to name collision while building time-series query.
## 3.10.1
Released 2017-08-04
Bug fixes:
- Exclude Infinities & NaNs from ramps #719.
- Fixed issue in time-series when aggregation starts at 1970-01-01 (epoch) #720.
## 3.10.0
Released 2017-08-03
Announcements:
- Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698.
- Support special numeric values (±Infinity, NaN) for json responses #706
## 3.9.8
Released 2017-07-21
- Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2).
## 3.9.7
Released 2017-07-20
Bug fixes:
- Respond with 204 (No content) when vector tile has no data #712
Announcements:
- Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2)
## 3.9.6
Released 2017-07-11
- Dataviews: support for aggregation in search results #708
## 3.9.5
Released 2017-06-27
- Dataviews: support special numeric values (±Infinity, NaN) #700
## 3.9.4
Released 2017-06-22
Announcements:
- Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6).
## 3.9.3
Released 2017-06-16
Announcements:
- Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5).
## 3.9.2
Released 2017-06-16
Announcements:
- Upgrades camshaft to [0.55.4](https://github.com/CartoDB/camshaft/releases/tag/0.55.4).
## 3.9.1
Released 2017-06-06
Announcements:
- Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3).
## 3.9.0
Released 2017-05-31
Announcements:
- Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1).
- Add support to retrieve info about layer stats in map instantiation.
- Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2).
- Remove promise polyfill from turbo-carto adapter
## 3.8.0
Released 2017-05-22
Announcements:
- Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0).
- Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1)
## 3.7.1
Released 2017-05-18
Bug fixes:
- Fix buffersize assignment when is not defined in requested mapconfig.
## 3.7.0
Released 2017-05-18
Announcements:
- Manage multiple values of buffer-size for different formats
- Upgrades windshaft to [3.2.0](https://github.com/CartoDB/windshaft/releases/tag/3.2.0).
## 3.6.6
Released 2017-05-11
Announcements:
- Upgrades camshaft to [0.54.4](https://github.com/CartoDB/camshaft/releases/tag/0.54.4).
## 3.6.5
Released 2017-05-09
Announcements:
- Upgrades camshaft to [0.54.3](https://github.com/CartoDB/camshaft/releases/tag/0.54.3).
## 3.6.4
Released 2017-05-05
Announcements:
- Upgrade cartodb-psql to [0.8.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.8.0).
- Upgrades camshaft to [0.54.2](https://github.com/CartoDB/camshaft/releases/tag/0.54.2).
- Upgrades windshaft to [3.1.2](https://github.com/CartoDB/windshaft/releases/tag/3.1.2).
## 3.6.3
Released 2017-04-25
Announcements:
- Upgrades windshaft to [3.1.1](https://github.com/CartoDB/windshaft/releases/tag/3.1.1).
## 3.6.2
Released 2017-04-24
Announcements:
- Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3).
## 3.6.1
Released 2017-04-24
Announcements:
- Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1).
## 3.6.0
Released 2017-04-20
Announcements:
- Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0).
## 3.5.1
Released 2017-04-11
Announcements:
- Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1).
## 3.5.0
Released 2017-04-10
Bug fixes:
- Fix invalidation of cache for maps with analyses #638.
Announcements:
- Upgrades camshaft to [0.53.0](https://github.com/CartoDB/camshaft/releases/tag/0.53.0).
## 3.4.0
Released 2017-04-03
Announcements:
- Upgrades camshaft to [0.51.0](https://github.com/CartoDB/camshaft/releases/tag/0.51.0).
## 3.3.0
Released 2017-04-03
New features:
- Static map endpoints allow specifying the layers to render #653.
## 3.2.0
Released 2017-03-30
Announcements:
- Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0).
- Active GC interval.
## 3.1.1
Released 2017-03-23

14
app.js
View File

@ -144,3 +144,17 @@ process.on('SIGHUP', function() {
process.on('uncaughtException', function(err) {
global.logger.error('Uncaught exception: ' + err.stack);
});
if (global.gc) {
var gcInterval = Number.isFinite(global.environment.gc_interval) ?
global.environment.gc_interval :
10000;
if (gcInterval > 0) {
setInterval(function gcForcedCycle() {
var start = Date.now();
global.gc();
global.statsClient.timing('windshaft.gc', Date.now() - start);
}, gcInterval);
}
}

6
config/environments/development.js.example
View File

@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '^(.*)\\.localhost'
@ -321,8 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: true
layerStats: true
}
};

5
config/environments/production.js.example
View File

@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '^(.*)\\.cartodb\\.com$'
@ -321,7 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: false
layerStats: false
}
};

5
config/environments/staging.js.example
View File

@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '^(.*)\\.cartodb\\.com$'
@ -321,7 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: true
layerStats: true
}
};

5
config/environments/test.js.example
View File

@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '(.*)'
@ -315,7 +318,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: true
layerStats: true
}
};

4
docs/static_maps_api.md
View File

@ -150,6 +150,10 @@ It is important to note that generated images are cached from the live data refe
* Image resolution is set to 72 DPI
* JPEG quality is 85%
* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
* If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code:
{% highlight javascript %}attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors, &copy; <a href="https://carto.com/attributions">CARTO</a>
{% endhighlight %}
## Examples

8
lib/cartodb/api/auth_api.js
View File

@ -95,9 +95,7 @@ AuthApi.prototype.authorize = function(req, callback) {
self.authorizedByAPIKey(user, req, this);
},
function checkApiKey(err, authorized){
if (req.profiler) {
req.profiler.done('authorizedByAPIKey');
}
req.profiler.done('authorizedByAPIKey');
assert.ifError(err);
// if not authorized by api_key, continue
@ -131,9 +129,7 @@ AuthApi.prototype.authorize = function(req, callback) {
}
self.pgConnection.setDBAuth(user, req.params, function(err) {
if (req.profiler) {
req.profiler.done('setDBAuth');
}
req.profiler.done('setDBAuth');
callback(err, true); // authorized (or error)
});
}

63
lib/cartodb/api/user_limits_api.js
View File

@ -1,3 +1,5 @@
var step = require('step');
/**
*
* @param metadataBackend
@ -13,16 +15,65 @@ function UserLimitsApi(metadataBackend, options) {
module.exports = UserLimitsApi;
UserLimitsApi.prototype.getRenderLimits = function (username, callback) {
UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) {
var self = this;
this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) {
var limits = {
cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
render: self.options.limits.render || 0
};
self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) {
if (err) {
return callback(err);
}
return callback(null, {
cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
render: renderLimit || self.options.limits.render || 0
});
if (timeoutRenderLimit && timeoutRenderLimit.render) {
if (Number.isFinite(timeoutRenderLimit.render)) {
limits.render = timeoutRenderLimit.render;
}
}
return callback(null, limits);
});
};
UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
var self = this;
step(
function isAuthorized() {
var next = this;
if (!apiKey) {
return next(null, false);
}
self.metadataBackend.getUserMapKey(username, function (err, userApiKey) {
if (err) {
return next(err);
}
return next(null, userApiKey === apiKey);
});
},
function getUserTimeoutRenderLimits(err, authorized) {
var next = this;
if (err) {
return next(err);
}
self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) {
if (err) {
return next(err);
}
next(null, {
render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
});
});
},
callback
);
};

90
lib/cartodb/backends/dataview.js
View File

@ -43,53 +43,19 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
ownFilter = !!ownFilter;
var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(mapConfig.obj().layers, function(l) {
return l.options.source && (l.options.source.id === sourceId);
});
var queryRewriteData = layer && layer.options.query_rewrite_data;
if (queryRewriteData && dataviewDefinition.node.type === 'source') {
queryRewriteData = _.extend({}, queryRewriteData, {
filters: dataviewDefinition.node.filters,
unfiltered_query: dataviewDefinition.sql.own_filter_on
});
}
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
query = bboxFilter.sql(query);
if ( queryRewriteData ) {
var bbox_filter_definition = {
type: 'bbox',
options: {
column: 'the_geom_webmercator',
srid: 3857
},
params: {
bbox: params.bbox
}
};
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
}
}
var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
);
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
function castNumbers(overrides, val, k) {
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
}
overrides[k] = +val;
return overrides;
},
{ownFilter: ownFilter}
);
var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
dataview.getResult(pg, overrideParams, this);
dataview.getResult(pg, getOverrideParams(params, ownFilter), this);
},
function returnCallback(err, result) {
return callback(err, result);
@ -97,6 +63,56 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
);
};
function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(mapConfig.obj().layers, function(l) {
return l.options.source && (l.options.source.id === sourceId);
});
var queryRewriteData = layer && layer.options.query_rewrite_data;
if (queryRewriteData && dataviewDefinition.node.type === 'source') {
queryRewriteData = _.extend({}, queryRewriteData, {
filters: dataviewDefinition.node.filters,
unfiltered_query: dataviewDefinition.sql.own_filter_on
});
}
if (params.bbox && queryRewriteData) {
var bbox_filter_definition = {
type: 'bbox',
options: {
column: 'the_geom_webmercator',
srid: 3857
},
params: {
bbox: params.bbox
}
};
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
}
return queryRewriteData;
}
function getOverrideParams(params, ownFilter) {
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'),
function castNumbers(overrides, val, k) {
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
}
overrides[k] = +val;
return overrides;
},
{ownFilter: ownFilter}
);
// validation will be delegated to the proper dataview
if (params.aggregation !== undefined) {
overrideParams.aggregation = params.aggregation;
}
return overrideParams;
}
DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
var dataviewName = params.dataviewName;

16
lib/cartodb/backends/layer-stats/empty-layer-stats.js Normal file
View File

@ -0,0 +1,16 @@
function EmptyLayerStats(types) {
this._types = types || {};
}
EmptyLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
EmptyLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
setImmediate(function() {
callback(null, {});
});
};
module.exports = EmptyLayerStats;

23
lib/cartodb/backends/layer-stats/factory.js Normal file
View File

@ -0,0 +1,23 @@
var LayerStats = require('./layer-stats');
var EmptyLayerStats = require('./empty-layer-stats');
var MapnikLayerStats = require('./mapnik-layer-stats');
var TorqueLayerStats = require('./torque-layer-stats');
module.exports = function LayerStatsFactory(type) {
var layerStatsIterator = [];
var selectedType = type || 'ALL';
if (selectedType === 'ALL') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true }));
layerStatsIterator.push(new MapnikLayerStats());
layerStatsIterator.push(new TorqueLayerStats());
} else if (selectedType === 'mapnik') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true }));
layerStatsIterator.push(new MapnikLayerStats());
} else if (selectedType === 'torque') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true }));
layerStatsIterator.push(new TorqueLayerStats());
}
return new LayerStats(layerStatsIterator);
};

45
lib/cartodb/backends/layer-stats/layer-stats.js Normal file
View File

@ -0,0 +1,45 @@
var queue = require('queue-async');
function LayerStats(layerStatsIterator) {
this.layerStatsIterator = layerStatsIterator;
}
LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) {
var self = this;
var stats = [];
if (!mapConfig.getLayers().length) {
return callback(null, stats);
}
var metaQueue = queue(mapConfig.getLayers().length);
mapConfig.getLayers().forEach(function (layer, layerId) {
var layerType = mapConfig.layerType(layerId);
for (var i = 0; i < self.layerStatsIterator.length; i++) {
if (self.layerStatsIterator[i].is(layerType)) {
var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]);
metaQueue.defer(getStats, layer, dbConnection);
break;
}
}
});
metaQueue.awaitAll(function (err, results) {
if (err) {
return callback(err);
}
if (!results) {
return callback(null, null);
}
mapConfig.getLayers().forEach(function (layer, layerIndex) {
stats[layerIndex] = results[layerIndex];
});
return callback(err, stats);
});
};
module.exports = LayerStats;

28
lib/cartodb/backends/layer-stats/mapnik-layer-stats.js Normal file
View File

@ -0,0 +1,28 @@
var queryUtils = require('../../utils/query-utils');
function MapnikLayerStats () {
this._types = {
mapnik: true,
cartodb: true
};
}
MapnikLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
MapnikLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
// This query would gather stats for postgresql table if not exists
dbConnection.query(queryRowCountSql, function (err, res) {
if (err) {
return callback(null, {estimatedFeatureCount: -1});
} else {
// We decided that the relation is 1 row == 1 feature
return callback(null, {estimatedFeatureCount: res.rows[0].rows});
}
});
};
module.exports = MapnikLayerStats;

16
lib/cartodb/backends/layer-stats/torque-layer-stats.js Normal file
View File

@ -0,0 +1,16 @@
function TorqueLayerStats() {
this._types = {
torque: true
};
}
TorqueLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
TorqueLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
return callback(null, {});
};
module.exports = TorqueLayerStats;

16
lib/cartodb/backends/stats.js Normal file
View File

@ -0,0 +1,16 @@
var layerStats = require('./layer-stats/factory');
function StatsBackend() {
}
module.exports = StatsBackend;
StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) {
var enabledFeatures = global.environment.enabledFeatures;
var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false;
if (layerStatsEnabled) {
layerStats().getStats(mapConfig, dbConnection, callback);
} else {
return callback(null, []);
}
};

15
lib/cartodb/backends/template_maps.js
View File

@ -430,13 +430,17 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
_reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
function _replaceVars (str, params) {
//return _.template(str, params); // lazy way, possibly dangerous
// Construct regular expressions for each param
// Construct regular expressions for each param
Object.keys(params).forEach(function(k) {
str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
});
return str;
}
function isObject(val) {
return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val));
}
TemplateMaps.prototype.instance = function(template, params) {
var all_params = {};
var phold = template.placeholders || {};
@ -474,6 +478,13 @@ TemplateMaps.prototype.instance = function(template, params) {
// NOTE: we're deep-cloning the layergroup here
var layergroup = JSON.parse(JSON.stringify(template.layergroup));
if (layergroup.buffersize && isObject(layergroup.buffersize)) {
Object.keys(layergroup.buffersize).forEach(function(k) {
layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10);
});
}
for (var i=0; i<layergroup.layers.length; ++i) {
var lyropt = layergroup.layers[i].options;

14
lib/cartodb/backends/turbo-carto-postgres-datasource.js
View File

@ -10,15 +10,21 @@ function createTemplate(method) {
'max({{=it._column}}) max_val,',
'avg({{=it._column}}) avg_val,',
method,
'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL',
'AND',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float'
].join('\n'));
}
var methods = {
quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
quantiles: 'CDB_QuantileBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as quantiles',
equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
jenks: 'CDB_JenksBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as jenks',
headtails: 'CDB_HeadsTailsBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as headtails'
};
var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {

132
lib/cartodb/controllers/base.js
View File

@ -4,9 +4,6 @@ var _ = require('underscore');
var step = require('step');
var debug = require('debug')('windshaft:cartodb');
var LZMA = require('lzma').LZMA;
var lzmaWorker = new LZMA();
// Whitelist query parameters and attach format
var REQUEST_QUERY_PARAMS_WHITELIST = [
'config',
@ -17,16 +14,8 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
'zoom',
'lon',
'lat',
// widgets & filters
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'bins', // number
'start', // number
'end', // number
'column_type', // string
// widgets search
'q'
// analysis
'filters' // json
];
function BaseController(authApi, pgConnection) {
@ -36,7 +25,7 @@ function BaseController(authApi, pgConnection) {
module.exports = BaseController;
// jshint maxcomplexity:9
// jshint maxcomplexity:8
/**
* Whitelist input and get database name & default geometry type from
* subdomain/user metadata held in CartoDB Redis
@ -46,38 +35,11 @@ module.exports = BaseController;
BaseController.prototype.req2params = function(req, callback){
var self = this;
if ( req.query.lzma ) {
// Decode (from base64)
var lzma = new Buffer(req.query.lzma, 'base64')
.toString('binary')
.split('')
.map(function(c) {
return c.charCodeAt(0) - 128;
});
// Decompress
lzmaWorker.decompress(
lzma,
function(result) {
if (req.profiler) {
req.profiler.done('lzma');
}
try {
delete req.query.lzma;
_.extend(req.query, JSON.parse(result));
self.req2params(req, callback);
} catch (err) {
req.profiler.done('req2params');
callback(new Error('Error parsing lzma as JSON: ' + err));
}
}
);
return;
var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
if (Array.isArray(req.context.allowedQueryParams)) {
allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams);
}
req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
req.query = _.pick(req.query, allowedQueryParams);
req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
var user = req.context.user;
@ -115,18 +77,14 @@ BaseController.prototype.req2params = function(req, callback){
// bring all query values onto req.params object
_.extend(req.params, req.query);
if (req.profiler) {
req.profiler.done('req2params.setup');
}
req.profiler.done('req2params.setup');
step(
function getPrivacy(){
self.authApi.authorize(req, this);
},
function validateAuthorization(err, authorized) {
if (req.profiler) {
req.profiler.done('authorize');
}
req.profiler.done('authorize');
assert.ifError(err);
if(!authorized) {
err = new Error("Sorry, you are unauthorized (permission denied)");
@ -167,9 +125,7 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
res.set('X-Served-By-DB-Host', req.params.dbhost);
}
if (req.profiler) {
res.set('X-Tiler-Profiler', req.profiler.toJSONString());
}
res.set('X-Tiler-Profiler', req.profiler.toJSONString());
if (headers) {
res.set(headers);
@ -187,26 +143,31 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
res.send(body);
}
if (req.profiler) {
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
debug("error sending profiling stats: " + err);
}
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
debug("error sending profiling stats: " + err);
}
};
// jshint maxcomplexity:6
BaseController.prototype.sendError = function(req, res, err, label) {
var allErrors = Array.isArray(err) ? err : [err];
allErrors = populateTimeoutErrors(allErrors);
label = label || 'UNKNOWN';
err = allErrors[0] || new Error(label);
allErrors[0] = err;
var statusCode = findStatusCode(err);
if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
statusCode = 204;
}
debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
// If a callback was requested, force status to 200
@ -231,6 +192,18 @@ function stripConnectionInfo(message) {
.replace(/is the server.*encountered/im, 'encountered');
}
var ERROR_INFO_TO_EXPOSE = {
message: true,
layer: true,
type: true,
analysis: true,
subtype: true
};
function shouldBeExposed (prop) {
return !!ERROR_INFO_TO_EXPOSE[prop];
}
function errorMessage(err) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var message = (_.isString(err) ? err : err.message) || 'Unknown error';
@ -249,7 +222,7 @@ function errorMessageWithContext(err) {
for (var prop in err) {
// type & message are properties from Error's prototype and will be skipped
if (err.hasOwnProperty(prop)) {
if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
error[prop] = err[prop];
}
}
@ -291,5 +264,38 @@ function statusFromErrorMessage(errMsg) {
statusCode = 404;
}
}
return statusCode;
}
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function populateTimeoutErrors (errors) {
return errors.map(function (error) {
if (isRenderTimeoutError(error)) {
error.subtype = 'render';
}
if (isDatasourceTimeoutError(error)) {
error.subtype = 'datasource';
}
if (isTimeoutError(error)) {
error.message = 'You are over platform\'s limits. Please contact us to know more details';
error.type = 'limit';
error.http_status = 429;
}
return error;
});
}

83
lib/cartodb/controllers/layergroup.js
View File

@ -6,6 +6,7 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var allowQueryParams = require('../middleware/allow-query-params');
var DataviewBackend = require('../backends/dataview');
var AnalysisStatusBackend = require('../backends/analysis-status');
@ -67,28 +68,62 @@ LayergroupController.prototype.register = function(app) {
this.attributes.bind(this));
app.get(app.base_url_mapconfig +
'/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware,
'/static/center/:token/:z/:lat/:lng/:width/:height.:format',
cors(), userMiddleware, allowQueryParams(['layer']),
this.center.bind(this));
app.get(app.base_url_mapconfig +
'/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
'/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
cors(), userMiddleware, allowQueryParams(['layer']),
this.bbox.bind(this));
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
app.get(app.base_url_mapconfig +
'/:token/dataview/:dataviewName', cors(), userMiddleware,
this.dataview.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:dataviewName', cors(), userMiddleware,
this.dataview.bind(this));
app.get(app.base_url_mapconfig +
'/:token/dataview/:dataviewName/search', cors(), userMiddleware,
this.dataviewSearch.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware,
this.dataviewSearch.bind(this));
var allowedDataviewQueryParams = [
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'start', // number
'end', // number
'column_type', // string
'bins', // number
'aggregation', //string
'offset', // number
'q' // widgets search
];
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataview.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataview.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataviewSearch.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataviewSearch.bind(this)
);
app.get(app.base_url_mapconfig +
'/:token/analysis/node/:nodeId', cors(), userMiddleware,
@ -250,7 +285,9 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
grid_json: true,
json_torque: true,
torque_json: true,
png: true
png: true,
png32: true,
mvt: true
};
var formatStat = 'invalid';
@ -388,13 +425,15 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
function getSQL(err, mapConfig) {
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
var queries = [];
mapConfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
queries.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
return queries.length ? queries.join(';') : null;
},

62
lib/cartodb/controllers/map.js
View File

@ -20,6 +20,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
/**
* @param {AuthApi} authApi
* @param {PgConnection} pgConnection
@ -30,10 +31,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {MapConfigAdapter} mapConfigAdapter
* @param {StatsBackend} statsBackend
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) {
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
statsBackend) {
BaseController.call(this, authApi, pgConnection);
@ -47,6 +50,8 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.mapConfigAdapter = mapConfigAdapter;
this.resourceLocator = new ResourceLocator(global.environment);
this.statsBackend = statsBackend;
}
util.inherits(MapController, BaseController);
@ -87,9 +92,7 @@ MapController.prototype.createPost = function(req, res) {
};
MapController.prototype.instantiate = function(req, res) {
if (req.profiler) {
req.profiler.start('windshaft-cartodb.instance_template_post');
}
req.profiler.start('windshaft-cartodb.instance_template_post');
this.instantiateTemplate(req, res, function prepareTemplateParams(callback) {
if (!req.is('application/json')) {
@ -100,9 +103,7 @@ MapController.prototype.instantiate = function(req, res) {
};
MapController.prototype.jsonp = function(req, res) {
if (req.profiler) {
req.profiler.start('windshaft-cartodb.instance_template_get');
}
req.profiler.start('windshaft-cartodb.instance_template_get');
this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) {
var err = null;
@ -218,7 +219,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
var mapConfigProvider;
var mapConfig;
step(
function setupParams(){
self.req2params(req, this);
@ -253,7 +253,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
},
function afterLayergroupCreate(err, layergroup) {
assert.ifError(err);
self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this);
self.afterLayergroupCreate(req, res, mapConfig, layergroup,
mapConfigProvider.analysesResults,
this);
},
function finishTemplateInstantiation(err, layergroup) {
if (err) {
@ -276,7 +278,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
);
};
MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) {
MapController.prototype.afterLayergroupCreate =
function(req, res, mapconfig, layergroup, analysesResults, callback) {
var self = this;
var username = req.context.user;
@ -303,21 +306,26 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
// take place before proceeding. Error will be logged
// asynchronously
this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) {
if (req.profiler) {
req.profiler.done('incMapviewCount');
}
req.profiler.done('incMapviewCount');
if ( err ) {
global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
}
done();
});
var sql = mapconfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
var sql = [];
mapconfig.getLayers().forEach(function(layer) {
sql.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
sql.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
var dbName = req.params.dbname;
var layergroupId = layergroup.layergroupid;
var dbConnection;
step(
function getPgConnection() {
@ -325,12 +333,11 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
},
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
dbConnection = connection;
QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
req.profiler.done('queryTablesAndLastUpdated');
assert.ifError(err);
// feed affected tables cache so it can be reused from, for instance, layergroup controller
self.layergroupAffectedTables.set(dbName, layergroupId, result);
@ -354,6 +361,21 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
return null;
},
function fetchLayersStats(err) {
assert.ifError(err);
var next = this;
self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) {
if (err) {
return next(err);
}
if (layersStats.length > 0) {
layergroup.metadata.layers.forEach(function (layer, index) {
layer.meta.stats = layersStats[index];
});
}
return next();
});
},
function finish(err) {
done(err);
}

12
lib/cartodb/controllers/named_maps.js
View File

@ -8,6 +8,7 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var allowQueryParams = require('../middleware/allow-query-params');
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
@ -32,6 +33,7 @@ NamedMapsController.prototype.register = function(app) {
app.get(app.base_url_mapconfig +
'/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
this.staticMap.bind(this));
};
@ -100,9 +102,7 @@ NamedMapsController.prototype.tile = function(req, res) {
self.tileBackend.getTile(namedMapProvider, req.params, this);
},
function handleImage(err, tile, headers, stats) {
if (req.profiler) {
req.profiler.add(stats);
}
req.profiler.add(stats);
if (err) {
self.sendError(req, res, err, 'NAMED_MAP_TILE');
} else {
@ -176,10 +176,8 @@ NamedMapsController.prototype.staticMap = function(req, res) {
});
},
function handleImage(err, image, headers, stats) {
if (req.profiler) {
req.profiler.done('render-' + format);
req.profiler.add(stats || {});
}
req.profiler.done('render-' + format);
req.profiler.add(stats || {});
if (err) {
self.sendError(req, res, err, 'STATIC_VIZ_MAP');

12
lib/cartodb/controllers/named_maps_admin.js
View File

@ -91,9 +91,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
NamedMapsAdminController.prototype.retrieve = function(req, res) {
var self = this;
if (req.profiler) {
req.profiler.start('windshaft-cartodb.get_template');
}
req.profiler.start('windshaft-cartodb.get_template');
var cdbuser = req.context.user;
var tpl_id;
@ -127,9 +125,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
NamedMapsAdminController.prototype.destroy = function(req, res) {
var self = this;
if (req.profiler) {
req.profiler.start('windshaft-cartodb.delete_template');
}
req.profiler.start('windshaft-cartodb.delete_template');
var cdbuser = req.context.user;
var tpl_id;
@ -154,9 +150,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
NamedMapsAdminController.prototype.list = function(req, res) {
var self = this;
if ( req.profiler ) {
req.profiler.start('windshaft-cartodb.get_template_list');
}
req.profiler.start('windshaft-cartodb.get_template_list');
var cdbuser = req.context.user;

9
lib/cartodb/middleware/allow-query-params.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = function allowQueryParams(params) {
if (!Array.isArray(params)) {
throw new Error('allowQueryParams must receive an Array of params');
}
return function allowQueryParamsMiddleware(req, res, next) {
req.context.allowedQueryParams = params;
next();
};
};

30
lib/cartodb/middleware/lzma.js Normal file
View File

@ -0,0 +1,30 @@
'use strict';
var LZMA = require('lzma').LZMA;
var lzmaWorker = new LZMA();
module.exports = function lzmaMiddleware(req, res, next) {
if (!req.query.hasOwnProperty('lzma')) {
return next();
}
// Decode (from base64)
var lzma = new Buffer(req.query.lzma, 'base64')
.toString('binary')
.split('')
.map(function(c) {
return c.charCodeAt(0) - 128;
});
// Decompress
lzmaWorker.decompress(lzma, function(result) {
try {
delete req.query.lzma;
Object.assign(req.query, JSON.parse(result));
next();
} catch (err) {
next(new Error('Error parsing lzma as JSON: ' + err));
}
});
};

140
lib/cartodb/models/dataview/aggregation.js
View File

@ -5,11 +5,32 @@ var debug = require('debug')('windshaft:widget:aggregation');
var dot = require('dot');
dot.templateSettings.strip = false;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var summaryQueryTpl = dot.template([
'summary AS (',
' SELECT',
' count(1) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' {{?it._isFloatColumn}},sum(',
' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1',
' ELSE 0',
' END',
' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')'
].join('\n'));
@ -18,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' FROM filtered_source',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
@ -44,22 +65,25 @@ var categoriesSummaryCountQueryTpl = dot.template([
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
' count, categories_count',
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,',
' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n'));
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC'
].join('\n'));
@ -84,7 +108,7 @@ var TYPE = 'aggregation';
}
}
*/
function Aggregation(query, options) {
function Aggregation(query, options, queries) {
if (!_.isString(options.column)) {
throw new Error('Aggregation expects `column` in widget options');
}
@ -108,9 +132,11 @@ function Aggregation(query, options) {
BaseWidget.apply(this);
this.query = query;
this.queries = queries;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
}
Aggregation.prototype = new BaseWidget();
@ -119,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var _query = this.query;
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
@ -140,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) {
].join('\n');
} else {
aggregationSql = [
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
@ -155,30 +208,38 @@ Aggregation.prototype.sql = function(psql, override, callback) {
return callback(null, aggregationSql);
};
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) {
return [
"WITH",
[
summaryQueryTpl({
_query: query,
_column: column
}),
rankedCategoriesQueryTpl({
_query: query,
_column: column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
categoriesSummaryMinMaxQueryTpl({
_query: query,
_column: column
}),
categoriesSummaryCountQueryTpl({
_query: query,
_column: column
})
].join(',\n')
].join('\n');
"WITH",
[
filteredQueryTpl({
_isFloatColumn: isFloatColumn,
_query: this.query,
_column: this.column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
summaryQueryTpl({
_isFloatColumn: isFloatColumn,
_query: query,
_column: column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
rankedCategoriesQueryTpl({
_query: query,
_column: column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
categoriesSummaryMinMaxQueryTpl({
_query: query,
_column: column
}),
categoriesSummaryCountQueryTpl({
_query: query,
_column: column
})
].join(',\n')
].join('\n');
};
var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
@ -193,6 +254,8 @@ Aggregation.prototype.format = function(result) {
var categories = [];
var count = 0;
var nulls = 0;
var nans = 0;
var infinities = 0;
var minValue = 0;
var maxValue = 0;
var categoriesCount = 0;
@ -202,12 +265,15 @@ Aggregation.prototype.format = function(result) {
var firstRow = result.rows[0];
count = firstRow.count;
nulls = firstRow.nulls_count;
nans = firstRow.nans_count;
infinities = firstRow.infinities_count;
minValue = firstRow.min_val;
maxValue = firstRow.max_val;
categoriesCount = firstRow.categories_count;
result.rows.forEach(function(row) {
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val',
'max_val', 'categories_count', 'nans_count', 'infinities_count'));
});
}
@ -215,6 +281,8 @@ Aggregation.prototype.format = function(result) {
aggregation: this.aggregation,
count: count,
nulls: nulls,
nans: nans,
infinities: infinities,
min: minValue,
max: maxValue,
categoriesCount: categoriesCount,
@ -253,6 +321,8 @@ Aggregation.prototype.search = function(psql, userQuery, callback) {
var self = this;
var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
var _value = this.aggregation !== 'count' && this.aggregationColumn ?
this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)';
// TODO unfiltered will be wrong as filters are already applied at this point
var query = searchQueryTpl({
@ -265,7 +335,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) {
_searchFiltered: filterCategoriesQueryTpl({
_query: this.query,
_column: this.column,
_value: 'count(1)',
_value: _value,
_userQuery: _userQuery
})
});

47
lib/cartodb/models/dataview/base.js
View File

@ -1,3 +1,6 @@
var dot = require('dot');
dot.templateSettings.strip = false;
function BaseDataview() {}
module.exports = BaseDataview;
@ -5,8 +8,11 @@ module.exports = BaseDataview;
BaseDataview.prototype.getResult = function(psql, override, callback) {
var self = this;
this.sql(psql, override, function(err, query) {
psql.query(query, function(err, result) {
if (err) {
return callback(err);
}
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
}
@ -24,3 +30,42 @@ BaseDataview.prototype.getResult = function(psql, override, callback) {
BaseDataview.prototype.search = function(psql, userQuery, callback) {
return callback(null, this.format({ rows: [] }));
};
var FLOAT_OIDS = {
700: true,
701: true,
1700: true
};
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1'
);
BaseDataview.prototype.getColumnType = function (psql, column, query, callback) {
var readOnlyTransaction = true;
var columnTypeQuery = columnTypeQueryTpl({
column: column, query: query
});
psql.query(columnTypeQuery, function(err, result) {
if (err) {
return callback(err);
}
var pgType = result.rows[0].pg_typeof;
callback(null, getPGTypeName(pgType));
}, readOnlyTransaction);
};
function getPGTypeName (pgType) {
return {
float: FLOAT_OIDS.hasOwnProperty(pgType),
date: DATE_OIDS.hasOwnProperty(pgType)
};
}

43
lib/cartodb/models/dataview/formula.js
View File

@ -7,9 +7,19 @@ dot.templateSettings.strip = false;
var formulaQueryTpl = dot.template([
'SELECT',
'{{=it._operation}}({{=it._column}}) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
' {{=it._operation}}({{=it._column}}) AS result,',
' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n'));
var VALID_OPERATIONS = {
@ -31,7 +41,7 @@ var TYPE = 'formula';
}
}
*/
function Formula(query, options) {
function Formula(query, options, queries) {
if (!_.isString(options.operation)) {
throw new Error('Formula expects `operation` in widget options');
}
@ -47,8 +57,10 @@ function Formula(query, options) {
BaseWidget.apply(this);
this.query = query;
this.queries = queries;
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
}
Formula.prototype = new BaseWidget();
@ -57,14 +69,27 @@ Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var _query = this.query;
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var formulaSql = formulaQueryTpl({
_query: _query,
_isFloatColumn: this._isFloatColumn,
_query: this.query,
_operation: this.operation,
_column: this.column
});
@ -78,13 +103,17 @@ Formula.prototype.format = function(result) {
var formattedResult = {
operation: this.operation,
result: 0,
nulls: 0
nulls: 0,
nans: 0,
infinities: 0
};
if (result.rows.length) {
formattedResult.operation = this.operation;
formattedResult.result = result.rows[0].result;
formattedResult.nulls = result.rows[0].nulls_count;
formattedResult.nans = result.rows[0].nans_count;
formattedResult.infinities = result.rows[0].infinities_count;
}
return formattedResult;

559
lib/cartodb/models/dataview/histogram.js
View File

@ -5,108 +5,289 @@ var debug = require('debug')('windshaft:dataview:histogram');
var dot = require('dot');
dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
var dateIntervalQueryTpl = dot.template([
'WITH',
'__cdb_dates AS (',
' SELECT',
' MAX({{=it.column}}::timestamp) AS __cdb_end,',
' MIN({{=it.column}}::timestamp) AS __cdb_start',
' FROM ({{=it.query}}) __cdb_source',
'),',
'__cdb_interval_in_days AS (',
' SELECT' ,
' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days',
' FROM __cdb_dates',
'),',
'__cdb_interval_in_hours AS (',
' SELECT',
' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours',
' FROM __cdb_interval_in_days, __cdb_dates',
'),',
'__cdb_interval_in_minutes AS (',
' SELECT',
' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes',
' FROM __cdb_interval_in_hours, __cdb_dates',
'),',
'__cdb_interval_in_seconds AS (',
' SELECT',
' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds',
' FROM __cdb_interval_in_minutes, __cdb_dates',
')',
'SELECT',
' ROUND(__cdb_days / 365) AS year,',
' ROUND(__cdb_days / 90) AS quarter,',
' ROUND(__cdb_days / 30) AS month,',
' ROUND(__cdb_days / 7) AS week,',
' __cdb_days AS day,',
' __cdb_hours AS hour,',
' __cdb_minutes AS minute,',
' __cdb_seconds AS second',
'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds'
].join('\n'));
var MAX_INTERVAL_VALUE = 366;
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
var filteredQueryTpl = dot.template([
'__cdb_filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) __cdb_filtered_source_query',
' WHERE',
' {{=it._column}} IS NOT NULL',
' {{?it._isFloatColumn}}AND',
' {{=it._column}} != \'infinity\'::float',
' AND',
' {{=it._column}} != \'-infinity\'::float',
' AND',
' {{=it._column}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var basicsQueryTpl = dot.template([
'basics AS (',
'__cdb_basics AS (',
' SELECT',
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,',
' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
' FROM __cdb_filtered_source',
')'
].join(' \n'));
var overrideBasicsQueryTpl = dot.template([
'basics AS (',
'__cdb_basics AS (',
' SELECT',
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,',
' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
' FROM __cdb_filtered_source',
')'
].join('\n'));
var iqrQueryTpl = dot.template([
'iqrange AS (',
' SELECT max(quartile_max) - min(quartile_max) AS iqr',
'__cdb_iqrange AS (',
' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr',
' FROM (',
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile',
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
' FROM __cdb_filtered_source) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile',
' ) _cdb_iqr',
' ) __cdb_iqr',
')'
].join('\n'));
var binsQueryTpl = dot.template([
'bins AS (',
' SELECT CASE WHEN total_rows = 0 OR iqr = 0',
'__cdb_bins AS (',
' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0',
' THEN 1',
' ELSE GREATEST(',
' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),',
' LEAST(',
' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),',
' {{=it._maxBins}}',
' )',
' )',
' END AS bins_number',
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
' END AS __cdb_bins_number',
' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source',
' LIMIT 1',
')'
].join('\n'));
var overrideBinsQueryTpl = dot.template([
'bins AS (',
' SELECT {{=it._bins}} AS bins_number',
'__cdb_bins AS (',
' SELECT {{=it._bins}} AS __cdb_bins_number',
')'
].join('\n'));
var nullsQueryTpl = dot.template([
'nulls AS (',
'__cdb_nulls AS (',
' SELECT',
' count(*) AS nulls_count',
' FROM ({{=it._query}}) _cdb_histogram_nulls',
' count(*) AS __cdb_nulls_count',
' FROM ({{=it._query}}) __cdb_histogram_nulls',
' WHERE {{=it._column}} IS NULL',
')'
].join('\n'));
var infinitiesQueryTpl = dot.template([
'__cdb_infinities AS (',
' SELECT',
' count(*) AS __cdb_infinities_count',
' FROM ({{=it._query}}) __cdb_infinities_query',
' WHERE',
' {{=it._column}} = \'infinity\'::float',
' OR',
' {{=it._column}} = \'-infinity\'::float',
')'
].join('\n'));
var nansQueryTpl = dot.template([
'__cdb_nans AS (',
' SELECT',
' count(*) AS __cdb_nans_count',
' FROM ({{=it._query}}) __cdb_nans_query',
' WHERE {{=it._column}} = \'NaN\'::float',
')'
].join('\n'));
var histogramQueryTpl = dot.template([
'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,',
' nulls_count,',
' avg_val,',
' CASE WHEN min_val = max_val',
' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
' __cdb_bins_number AS bins_number,',
' __cdb_nulls_count AS nulls_count,',
' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,',
' __cdb_nans_count AS nans_count,{{?}}',
' __cdb_avg_val AS avg_val,',
' CASE WHEN __cdb_min_val = __cdb_max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
' ELSE GREATEST(',
' 1,',
' LEAST(',
' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),',
' __cdb_bins_number',
' )',
' ) - 1',
' END AS bin,',
' min({{=it._column}})::numeric AS min,',
' max({{=it._column}})::numeric AS max,',
' avg({{=it._column}})::numeric AS avg,',
' count(*) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
'WHERE {{=it._column}} IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,',
' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}',
'GROUP BY bin, bins_number, bin_width, nulls_count,',
' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
'ORDER BY bin'
].join('\n'));
var dateBasicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,',
' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
' min(date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' )) AS __cdb_start_date,',
' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,',
' count(1) AS __cdb_total_rows',
' FROM ({{=it._query}}) __cdb_basics_query',
')'
].join(' \n'));
var dateOverrideBasicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max({{=it._end}})::float AS __cdb_max_val,',
' min({{=it._start}})::float AS __cdb_min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
' min(',
' date_trunc(',
' \'{{=it._aggregation}}\',',
' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' )',
' ) AS __cdb_start_date,',
' max(',
' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' ) AS __cdb_end_date,',
' count(1) AS __cdb_total_rows',
' FROM ({{=it._query}}) __cdb_basics_query',
')'
].join(' \n'));
var dateBinsQueryTpl = dot.template([
'__cdb_bins AS (',
' SELECT',
' __cdb_bins_array,',
' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number',
' FROM (',
' SELECT',
' ARRAY(',
' SELECT GENERATE_SERIES(',
' __cdb_start_date::timestamptz,',
' __cdb_end_date::timestamptz,',
' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval',
' )',
' ) AS __cdb_bins_array',
' FROM __cdb_basics',
' ) __cdb_bins_array_query',
')'
].join('\n'));
var dateHistogramQueryTpl = dot.template([
'SELECT',
' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
' __cdb_bins_number AS bins_number,',
' __cdb_nulls_count AS nulls_count,',
' CASE WHEN __cdb_min_val = __cdb_max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(',
' WIDTH_BUCKET(',
' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',',
' __cdb_bins_array',
' ),',
' __cdb_bins_number',
' )) - 1',
' END AS bin,',
' min(',
' date_part(',
' \'epoch\', ',
' date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' ) AT TIME ZONE \'{{=it._offset}}\'',
' )',
' )::numeric AS timestamp,',
' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,',
' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
' count(*) AS freq',
'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls',
'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start',
'ORDER BY bin'
].join('\n'));
var TYPE = 'histogram';
/**
{
type: 'histogram',
options: {
column: 'name',
bins: 10 // OPTIONAL
}
Numeric histogram:
{
type: 'histogram',
options: {
column: 'name', // column data type: numeric
bins: 10 // OPTIONAL
}
}
Time series:
{
type: 'histogram',
options: {
column: 'date', // column data type: date
aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric)
offset: -7200 // OPTIONAL (UTC offset in seconds)
}
}
*/
function Histogram(query, options, queries) {
@ -118,6 +299,8 @@ function Histogram(query, options, queries) {
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
this.aggregation = options.aggregation;
this.offset = options.offset;
this._columnType = null;
}
@ -127,50 +310,55 @@ Histogram.prototype.constructor = Histogram;
module.exports = Histogram;
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
Histogram.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var self = this;
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.queries.no_filters
});
if (this._columnType === null) {
psql.query(columnTypeQuery, function(err, result) {
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
// assume numeric, will fail later
self._columnType = 'numeric';
if (!err && !!result.rows[0]) {
var pgType = result.rows[0].pg_typeof;
if (DATE_OIDS.hasOwnProperty(pgType)) {
self._columnType = 'date';
}
if (!err && !!type) {
self._columnType = Object.keys(type).find(function (key) {
return type[key];
});
}
self.sql(psql, override, callback);
}, true); // use read-only transaction
return null;
}
this._buildQuery(psql, override, callback);
};
Histogram.prototype.isDateHistogram = function (override) {
return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined);
};
Histogram.prototype._buildQuery = function (psql, override, callback) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.query;
if (this.isDateHistogram(override)) {
return this._buildDateHistogramQuery(psql, override, callback);
}
if (this._columnType === 'date') {
_column = columnCastTpl({column: _column});
}
var _query = this.query;
filteredQuery = filteredQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
});
var basicsQuery, binsQuery;
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
if (this._shouldOverride(override)) {
debug('overriding with %j', override);
basicsQuery = overrideBasicsQueryTpl({
_query: _query,
@ -190,7 +378,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
_column: _column
});
if (override && _.has(override, 'bins')) {
if (this._shouldOverrideBins(override)) {
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
@ -211,18 +399,34 @@ Histogram.prototype.sql = function(psql, override, callback) {
}
}
var cteSql = [
filteredQuery,
basicsQuery,
binsQuery,
nullsQueryTpl({
_query: _query,
_column: _column
})
];
var histogramSql = [
"WITH",
[
basicsQuery,
binsQuery,
nullsQueryTpl({
if (this._columnType === 'float') {
cteSql.push(
infinitiesQueryTpl({
_query: _query,
_column: _column
}),
nansQueryTpl({
_query: _query,
_column: _column
})
].join(',\n'),
);
}
var histogramSql = [
"WITH",
cteSql.join(',\n'),
histogramQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
})
@ -233,6 +437,143 @@ Histogram.prototype.sql = function(psql, override, callback) {
return callback(null, histogramSql);
};
Histogram.prototype._shouldOverride = function (override) {
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
};
Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};
var DATE_AGGREGATIONS = {
'auto': true,
'minute': true,
'hour': true,
'day': true,
'week': true,
'month': true,
'quarter': true,
'year': true
};
Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) {
var _column = this.column;
var _query = this.query;
var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset;
if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) {
return callback(new Error('Invalid aggregation value. Valid ones: ' +
Object.keys(DATE_AGGREGATIONS).join(', ')
));
}
if (_aggregation === 'auto') {
this.getAutomaticAggregation(psql, function (err, aggregation) {
if (err || aggregation === 'none') {
this.aggregation = 'day';
} else {
this.aggregation = aggregation;
}
override.aggregation = this.aggregation;
this._buildDateHistogramQuery(psql, override, callback);
}.bind(this));
return null;
}
var dateBasicsQuery;
if (override && _.has(override, 'start') && _.has(override, 'end')) {
dateBasicsQuery = dateOverrideBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_start: getBinStart(override),
_end: getBinEnd(override),
_offset: parseOffset(_offset, _aggregation)
});
} else {
dateBasicsQuery = dateBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
}
var dateBinsQuery = [
dateBinsQueryTpl({
_aggregation: _aggregation
})
].join(',\n');
var nullsQuery = nullsQueryTpl({
_query: _query,
_column: _column
});
var dateHistogramQuery = dateHistogramQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
var histogramSql = [
"WITH",
[
dateBasicsQuery,
dateBinsQuery,
nullsQuery
].join(',\n'),
dateHistogramQuery
].join('\n');
debug(histogramSql);
return callback(null, histogramSql);
};
Histogram.prototype.getAutomaticAggregation = function (psql, callback) {
var dateIntervalQuery = dateIntervalQueryTpl({
query: this.query,
column: this.column
});
debug(dateIntervalQuery);
psql.query(dateIntervalQuery, function (err, result) {
if (err) {
return callback(err);
}
var aggegations = result.rows[0];
var aggregation = Object.keys(aggegations)
.map(function (key) {
return {
name: key,
value: aggegations[key]
};
})
.reduce(function (closer, current) {
if (current.value > MAX_INTERVAL_VALUE) {
return closer;
}
var closerDiff = MAX_INTERVAL_VALUE - closer.value;
var currentDiff = MAX_INTERVAL_VALUE - current.value;
if (Number.isFinite(current.value) && closerDiff > currentDiff) {
return current;
}
return closer;
}, { name: 'none', value: -1 });
callback(null, aggregation.name);
});
};
Histogram.prototype.format = function(result, override) {
override = override || {};
var buckets = [];
@ -241,7 +582,12 @@ Histogram.prototype.format = function(result, override) {
var width = getWidth(override);
var binsStart = getBinStart(override);
var nulls = 0;
var infinities = 0;
var nans = 0;
var avg;
var timestampStart;
var aggregation;
var offset;
if (result.rows.length) {
var firstRow = result.rows[0];
@ -249,23 +595,60 @@ Histogram.prototype.format = function(result, override) {
width = firstRow.bin_width || width;
avg = firstRow.avg_val;
nulls = firstRow.nulls_count;
binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min;
timestampStart = firstRow.timestamp_start;
infinities = firstRow.infinities_count;
nans = firstRow.nans_count;
binsStart = populateBinStart(override, firstRow);
if (Number.isFinite(timestampStart)) {
aggregation = getAggregation(override, this.aggregation);
offset = getOffset(override, this.offset);
}
buckets = result.rows.map(function(row) {
return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
return _.omit(
row,
'bins_number',
'bin_width',
'nulls_count',
'infinities_count',
'nans_count',
'avg_val',
'timestamp_start'
);
});
}
return {
aggregation: aggregation,
offset: offset,
timestamp_start: timestampStart,
bin_width: width,
bins_count: binsCount,
bins_start: binsStart,
nulls: nulls,
infinities: infinities,
nans: nans,
avg: avg,
bins: buckets
};
};
function getAggregation(override, aggregation) {
return override && override.aggregation ? override.aggregation : aggregation;
}
function getOffset(override, offset) {
if (override && override.offset) {
return override.offset;
}
if (offset) {
return offset;
}
return 0;
}
function getBinStart(override) {
if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
return Math.min(override.start, override.end);
@ -295,6 +678,32 @@ function getWidth(override) {
return width;
}
function parseOffset(offset, aggregation) {
if (!offset) {
return '0';
}
if (aggregation === 'hour' || aggregation === 'minute') {
return '0';
}
var offsetInHours = Math.ceil(offset / 3600);
return '' + offsetInHours;
}
function populateBinStart(override, firstRow) {
var binStart;
if (firstRow.hasOwnProperty('timestamp')) {
binStart = firstRow.timestamp;
} else if (override.hasOwnProperty('start')) {
binStart = getBinStart(override);
} else {
binStart = firstRow.min;
}
return binStart;
}
Histogram.prototype.getType = function() {
return TYPE;
};

94
lib/cartodb/models/dataview/overviews/aggregation.js
View File

@ -1,14 +1,36 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../aggregation');
var debug = require('debug')('windshaft:widget:aggregation:overview');
var dot = require('dot');
dot.templateSettings.strip = false;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var summaryQueryTpl = dot.template([
'summary AS (',
' SELECT',
' sum(_feature_count) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' {{?it._isFloatColumn}},sum(',
' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1',
' ELSE 0',
' END',
' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')'
].join('\n'));
@ -17,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' FROM filtered_source',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
@ -36,40 +58,46 @@ var categoriesSummaryCountQueryTpl = dot.template([
' SELECT count(1) AS categories_count',
' FROM (',
' SELECT {{=it._column}} AS category',
' FROM ({{=it._query}}) _cdb_categories',
' FROM filtered_source',
' GROUP BY {{=it._column}}',
' ) _cdb_categories_count',
')'
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n'));
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC'
].join('\n'));
var CATEGORIES_LIMIT = 6;
function Aggregation(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.query = query;
this.queries = queries;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
}
Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
@ -78,27 +106,49 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var _query = this.rewrittenQuery(this.query);
var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
_aggregationColumn: _aggregationColumn
}),
categoriesSummaryMinMaxQueryTpl({
_query: _query,
@ -110,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
})
].join(',\n'),
aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
@ -120,15 +171,23 @@ Aggregation.prototype.sql = function(psql, override, callback) {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
_aggregationColumn: _aggregationColumn
}),
categoriesSummaryMinMaxQueryTpl({
_query: _query,
@ -140,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
})
].join(',\n'),
rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_limit: CATEGORIES_LIMIT
@ -147,6 +207,8 @@ Aggregation.prototype.sql = function(psql, override, callback) {
].join('\n');
}
debug(aggregationSql);
return callback(null, aggregationSql);
};

5
lib/cartodb/models/dataview/overviews/base.js
View File

@ -1,14 +1,15 @@
var _ = require('underscore');
var BaseDataview = require('../base');
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) {
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) {
this.BaseDataview = BaseDataview;
this.query = query;
this.queryOptions = queryOptions;
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
this.options = options;
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions);
this.queries = queries;
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries);
}
module.exports = BaseOverviewsDataview;

92
lib/cartodb/models/dataview/overviews/formula.js
View File

@ -1,34 +1,61 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overview');
var dot = require('dot');
dot.templateSettings.strip = false;
var formulaQueryTpls = {
'count': dot.template([
'SELECT',
'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'count': dot.template([
'SELECT',
'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
};
function Formula(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
this.queries = queries;
}
Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
@ -36,21 +63,38 @@ Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) {
Formula.prototype.sql = function (psql, override, callback) {
var self = this;
var formulaQueryTpl = formulaQueryTpls[this.operation];
if ( formulaQueryTpl ) {
if (formulaQueryTpl) {
// supported formula for use with overviews
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var formulaSql = formulaQueryTpl({
_query: this.rewrittenQuery(this.query),
_isFloatColumn: this._isFloatColumn,
_query: this.rewrittenQuery(this.query),
_operation: this.operation,
_column: this.column
_column: this.column
});
callback = callback || override;
debug(formulaSql);
return callback(null, formulaSql);
}
// default behaviour
return this.defaultSql(psql, override, callback);
};

147
lib/cartodb/models/dataview/overviews/histogram.js
View File

@ -1,23 +1,35 @@
var _ = require('underscore');
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../histogram');
var debug = require('debug')('windshaft:dataview:histogram:overview');
var dot = require('dot');
dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' WHERE',
' {{=it._column}} IS NOT NULL',
' {{?it._isFloatColumn}}AND',
' {{=it._column}} != \'infinity\'::float',
' AND',
' {{=it._column}} != \'-infinity\'::float',
' AND',
' {{=it._column}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var basicsQueryTpl = dot.template([
'basics AS (',
' SELECT',
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' FROM filtered_source',
')'
].join(' \n'));
@ -26,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([
' SELECT',
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' FROM filtered_source',
')'
].join('\n'));
@ -37,7 +49,7 @@ var iqrQueryTpl = dot.template([
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile',
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
' FROM filtered_source) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile',
' ) _cdb_iqr',
@ -56,7 +68,7 @@ var binsQueryTpl = dot.template([
' )',
' )',
' END AS bins_number',
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
' FROM basics, iqrange, filtered_source',
' LIMIT 1',
')'
].join('\n'));
@ -76,11 +88,34 @@ var nullsQueryTpl = dot.template([
')'
].join('\n'));
var infinitiesQueryTpl = dot.template([
'infinities AS (',
' SELECT',
' count(*) AS infinities_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE',
' {{=it._column}} = \'infinity\'::float',
' OR',
' {{=it._column}} = \'-infinity\'::float',
')'
].join('\n'));
var nansQueryTpl = dot.template([
'nans AS (',
' SELECT',
' count(*) AS nans_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE {{=it._column}} = \'NaN\'::float',
')'
].join('\n'));
var histogramQueryTpl = dot.template([
'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,',
' nulls_count,',
' {{?it._isFloatColumn}}infinities_count,',
' nans_count,{{?}}',
' avg_val,',
' CASE WHEN min_val = max_val',
' THEN 0',
@ -90,14 +125,14 @@ var histogramQueryTpl = dot.template([
' max({{=it._column}})::numeric AS max,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
' sum(_feature_count) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
'WHERE {{=it._column}} IS NOT NULL',
'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
'ORDER BY bin'
].join('\n'));
function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.query = query;
this.queries = queries;
@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram;
module.exports = Histogram;
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
Histogram.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var self = this;
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.rewrittenQuery(this.queries.no_filters)
});
if (this._columnType === null) {
psql.query(columnTypeQuery, function(err, result) {
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
// assume numeric, will fail later
self._columnType = 'numeric';
if (!err && !!result.rows[0]) {
var pgType = result.rows[0].pg_typeof;
if (DATE_OIDS.hasOwnProperty(pgType)) {
self._columnType = 'date';
}
if (!err && !!type) {
self._columnType = Object.keys(type).find(function (key) {
return type[key];
});
}
self.sql(psql, override, callback);
}, true); // use read-only transaction
@ -154,11 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) {
return this.defaultSql(psql, override, callback);
}
var histogramSql = this._buildQuery(override);
return callback(null, histogramSql);
};
Histogram.prototype._buildQuery = function (override) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.rewrittenQuery(this.query);
var basicsQuery, binsQuery;
filteredQuery = filteredQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
});
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
if (this._shouldOverride(override)) {
debug('overriding with %j', override);
basicsQuery = overrideBasicsQueryTpl({
_query: _query,
_column: _column,
@ -177,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
_column: _column
});
if (override && _.has(override, 'bins')) {
if (this._shouldOverrideBins(override)) {
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
@ -198,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) {
}
}
var cteSql = [
filteredQuery,
basicsQuery,
binsQuery,
nullsQueryTpl({
_query: _query,
_column: _column
})
];
var histogramSql = [
"WITH",
[
basicsQuery,
binsQuery,
nullsQueryTpl({
if (this._columnType === 'float') {
cteSql.push(
infinitiesQueryTpl({
_query: _query,
_column: _column
}),
nansQueryTpl({
_query: _query,
_column: _column
})
].join(',\n'),
);
}
var histogramSql = [
"WITH",
cteSql.join(',\n'),
histogramQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
})
].join('\n');
return callback(null, histogramSql);
debug(histogramSql);
return histogramSql;
};
Histogram.prototype._shouldOverride = function (override) {
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
};
Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};

4
lib/cartodb/models/dataview/overviews/list.js
View File

@ -1,8 +1,8 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../list');
function List(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
function List(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
}
List.prototype = Object.create(BaseOverviewsDataview.prototype);

5
lib/cartodb/models/filter/bbox.js
View File

@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([
].join('\n'));
var bboxFilterTpl = dot.template(
'{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'
'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))'
);
var LATITUDE_MAX_VALUE = 85.0511287798066;
@ -66,7 +66,8 @@ function getBoundingBoxes(west, south, east, north) {
bboxes.push([west, south, east, north]);
} else {
bboxes.push([west, south, 180, north]);
bboxes.push([-180, south, east % 180, north]);
// here we assume west,east have been adjusted => west >= -180 => east > 180
bboxes.push([-180, south, east - 360, north]);
}
return bboxes;

10
lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js
View File

@ -115,6 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
}
layer.options.sql = analysisSql;
layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode);
} else {
missingNodesErrors.push(
new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
@ -330,4 +331,13 @@ function AnalysisError(message) {
this.message = message;
}
function getAllAffectedTablesFromSourceNodes(node) {
var affectedTables = node.getAllInputNodes(function (node) {
return node.getType() === 'source';
}).reduce(function(list, node) {
return list.concat(node.getAffectedTables());
},[]);
return affectedTables;
}
require('util').inherits(AnalysisError, Error);

25
lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js Normal file
View File

@ -0,0 +1,25 @@
function MapConfigBufferSizeAdapter() {
this.formats = ['png', 'png32', 'mvt', 'grid.json'];
}
module.exports = MapConfigBufferSizeAdapter;
MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
if (!context.templateParams || !context.templateParams.buffersize) {
return callback(null, requestMapConfig);
}
this.formats.forEach(function (format) {
if (Number.isFinite(context.templateParams.buffersize[format])) {
if (requestMapConfig.buffersize === undefined) {
requestMapConfig.buffersize = {};
}
requestMapConfig.buffersize[format] = context.templateParams.buffersize[format];
}
});
setImmediate(function () {
callback(null, requestMapConfig);
});
};

1
lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js
View File

@ -43,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC
if (nestedNamedLayers.length > 0) {
var nestedNamedMapsError = new Error('Nested named layers are not allowed');
// nestedNamedMapsError.http_status = 400;
return done(nestedNamedMapsError);
}

7
lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js
View File

@ -4,13 +4,6 @@ var dot = require('dot');
dot.templateSettings.strip = false;
var queue = require('queue-async');
var PSQL = require('cartodb-psql');
/**
* cartodb-psql creates `global.Promise` as an empty constructor.
* However, `turbo-carto` relies on a polyfil that fails to create the polyfil
* as it finds `global.Promise` but it doesn't find `Promise.resolve`.
*/
global.Promise = global.Promise || function() {};
global.Promise.resolve = global.Promise.resolve || function() {};
var turboCarto = require('turbo-carto');
var SubstitutionTokens = require('../../../utils/substitution-tokens');

2
lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js
View File

@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, this);
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

2
lib/cartodb/models/mapconfig/provider/map-store-provider.js
View File

@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, this);
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

3
lib/cartodb/models/mapconfig/provider/named-map-provider.js
View File

@ -90,6 +90,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
},
function instantiateTemplate(err, templateParams) {
assert.ifError(err);
context.templateParams = templateParams;
return self.templateMaps.instance(self.template, templateParams);
},
function prepareAdapterMapConfig(err, requestMapConfig) {
@ -113,7 +114,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
function prepareContextLimits(err, _mapConfig) {
assert.ifError(err);
mapConfig = _mapConfig;
self.userLimitsApi.getRenderLimits(self.owner, this);
self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this);
},
function cacheAndReturnMapConfig(err, renderLimits) {
self.err = err;

55
lib/cartodb/server.js
View File

@ -4,6 +4,8 @@ var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var _ = require('underscore');
var lzmaMiddleware = require('./middleware/lzma');
var controller = require('./controllers');
var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
@ -35,12 +37,15 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin
var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
var MapConfigAdapter = require('./models/mapconfig/adapter');
var StatsBackend = require('./backends/stats');
module.exports = function(serverOptions) {
// Make stats client globally accessible
global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@ -115,8 +120,27 @@ module.exports = function(serverOptions) {
var onTileErrorStrategy;
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
if (err && err.message === 'Render timed out' && format === 'png') {
return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRasterFormat (format) {
return format === 'png' || format === 'jpg';
}
if (isTimeoutError(err) && isRasterFormat(format)) {
return callback(null, timeoutErrorTile, {
'Content-Type': 'image/png',
}, {});
} else {
return callback(err, tile, headers, stats);
}
@ -150,11 +174,14 @@ module.exports = function(serverOptions) {
var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
var statsBackend = new StatsBackend();
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var mapConfigAdapter = new MapConfigAdapter(
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
new MapConfigBufferSizeAdapter(),
new SqlWrapMapConfigAdapter(),
new DataviewsWidgetsAdapter(),
new AnalysisMapConfigAdapter(analysisBackend),
@ -207,7 +234,8 @@ module.exports = function(serverOptions) {
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
mapConfigAdapter
mapConfigAdapter,
statsBackend
).register(app);
new controller.NamedMaps(
@ -303,6 +331,25 @@ function bootstrap(opts) {
app.enable('jsonp callback');
app.disable('x-powered-by');
app.disable('etag');
// Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705
// See: http://expressjs.com/en/4x/api.html#app.set
app.set('json replacer', function (key, value) {
if (value !== value) {
return 'NaN';
}
if (value === Infinity) {
return 'Infinity';
}
if (value === -Infinity) {
return '-Infinity';
}
return value;
});
app.use(bodyParser.json());
app.use(function bootstrap$prepareRequestResponse(req, res, next) {
@ -319,6 +366,8 @@ function bootstrap(opts) {
next();
});
app.use(lzmaMiddleware);
// temporary measure until we upgrade to newer version expressjs so we can check err.status
app.use(function(err, req, res, next) {
if (err) {

26
lib/cartodb/utils/query-utils.js Normal file
View File

@ -0,0 +1,26 @@
function prepareQuery(sql) {
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
}
module.exports.extractTableNames = function extractTableNames(query) {
return [
'SELECT * FROM CDB_QueryTablesText($windshaft$',
prepareQuery(query),
'$windshaft$) as tablenames'
].join('');
};
module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
};

23
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "3.1.2",
"version": "3.12.11",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@ -16,14 +16,17 @@
"contributors": [
"Simon Tokumine <simon@vizzuality.com>",
"Javi Santana <jsantana@vizzuality.com>",
"Sandro Santilli <strk@vizzuality.com>"
"Sandro Santilli <strk@vizzuality.com>",
"Carlos Matallín <matallo@carto.com>",
"Daniel Garcia Aubert <dgaubert@carto.com>",
"Mario de Frutos <mario.defrutos@carto.com>"
],
"dependencies": {
"body-parser": "~1.14.0",
"camshaft": "0.50.3",
"cartodb-psql": "~0.7.1",
"camshaft": "0.58.1",
"cartodb-psql": "0.10.1",
"cartodb-query-tables": "0.2.0",
"cartodb-redis": "0.13.2",
"cartodb-redis": "0.14.0",
"debug": "~2.2.0",
"dot": "~1.0.2",
"express": "~4.13.3",
@ -38,20 +41,22 @@
"semver": "~5.3.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"turbo-carto": "0.19.0",
"turbo-carto": "0.19.2",
"underscore": "~1.6.0",
"windshaft": "3.0.1",
"windshaft": "3.3.2",
"yargs": "~5.0.0"
},
"devDependencies": {
"istanbul": "~0.4.3",
"jshint": "~2.6.0",
"mocha": "~1.21.4",
"jshint": "~2.9.4",
"mocha": "~3.4.1",
"moment": "~2.18.1",
"nock": "~2.11.0",
"redis": "~0.12.1",
"strftime": "~0.8.2"
},
"scripts": {
"lint": "jshint lib test",
"preinstall": "make pre-install",
"test": "make test-all"
},

658
test/acceptance/analysis/analysis-layers-use-cases.js
View File

@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client');
var dot = require('dot');
var debug = require('debug')('windshaft:cartodb:test');
describe('analysis-layers use cases', function() {
describe('analysis-layers use cases', function () {
var multitypeStyleTemplate = dot.template([
"#points['mapnik::geometry_type'=1] {",
" marker-fill-opacity: {{=it._opacity}};",
" marker-line-color: #FFF;",
" marker-line-width: 0.5;",
" marker-line-opacity: {{=it._opacity}};",
" marker-placement: point;",
" marker-type: ellipse;",
" marker-width: 8;",
" marker-fill: {{=it._color}};",
" marker-allow-overlap: true;",
"}",
"#lines['mapnik::geometry_type'=2] {",
" line-color: {{=it._color}};",
" line-width: 2;",
" line-opacity: {{=it._opacity}};",
"}",
"#polygons['mapnik::geometry_type'=3] {",
" polygon-fill: {{=it._color}};",
" polygon-opacity: {{=it._opacity}};",
" line-color: #FFF;",
" line-width: 0.5;",
" line-opacity: {{=it._opacity}};",
"}"
].join('\n'));
var multitypeStyleTemplate = dot.template(
`#points['mapnik::geometry_type'=1] {
marker-fill-opacity: {{=it._opacity}};
marker-line-color: #FFF;
marker-line-width: 0.5;
marker-line-opacity: {{=it._opacity}};
marker-placement: point;
marker-type: ellipse;
marker-width: 8;
marker-fill: {{=it._color}};
marker-allow-overlap: true;
}
#lines['mapnik::geometry_type'=2] {
line-color: {{=it._color}};
line-width: 2;
line-opacity: {{=it._opacity}};
}
#polygons['mapnik::geometry_type'=3] {
polygon-fill: {{=it._color}};
polygon-opacity: {{=it._opacity}};
line-color: #FFF;
line-width: 0.5;
line-opacity: {{=it._opacity}};
}`
);
function cartocss(color, opacity) {
@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analysis: analysis || []
analyses: analysis || []
};
}
function analysisDef(analysis) {
return JSON.stringify(analysis);
}
var DEFAULT_MULTITYPE_STYLE = cartocss();
var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 };
var pointInPolygonDef = {
id: 'a1',
type: 'point-in-polygon',
params: {
points_source: {
type: 'source',
params: {
query: 'select * from analysis_rent_listings'
}
},
polygons_source: {
type: 'buffer',
params: {
source: {
type: 'source',
params: {
query: 'select * from analysis_banks'
}
},
radius: 250
}
}
}
};
var bufferDef = {
id: 'b1',
type: 'buffer',
params: {
source: {
type: 'source',
params: {
query: 'select * from analysis_banks'
}
},
radius: 250
}
};
var useCases = [
{
desc: '1 mapnik layer',
@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() {
{
type: 'cartodb',
options: {
sql: "select * from analysis_rent_listings",
sql: 'select * from analysis_rent_listings',
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() {
{
type: 'cartodb',
options: {
sql: "select * from analysis_banks",
sql: 'select * from analysis_banks',
cartocss: cartocss('#2167AB'),
cartocss_version: '2.3.0'
}
@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() {
{
type: 'cartodb',
options: {
sql: "select * from analysis_rent_listings",
sql: 'select * from analysis_rent_listings',
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() {
{
type: 'cartodb',
options: {
sql: "select * from analysis_rent_listings",
sql: 'select * from analysis_rent_listings',
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
},
{
type: 'analysis',
type: 'cartodb',
options: {
def: analysisDef({
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from analysis_banks"
}
},
"radius": 250
}
}),
cartocss: cartocss('black', 0.5)
source: {
id: 'b1'
},
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
}
])
],
{},
[
bufferDef
]
)
},
{
@ -137,531 +169,115 @@ describe('analysis-layers use cases', function() {
{
type: 'cartodb',
options: {
sql: "select * from analysis_rent_listings",
sql: 'select * from analysis_rent_listings',
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
},
{
type: 'analysis',
type: 'cartodb',
options: {
def: analysisDef({
"type": "point-in-polygon",
"params": {
"pointsSource": {
"type": "source",
"params": {
"query": "select * from analysis_rent_listings"
}
},
"polygonsSource": {
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from analysis_banks"
}
},
"radius": 250
}
}
}
}),
cartocss: cartocss('green', 1.0)
source: {
id: 'a1'
},
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
}
])
],
{},
[
pointInPolygonDef
]
)
},
{
desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings',
mapConfig: mapConfig([
{
type: 'analysis',
options: {
def: analysisDef({
"type": "point-in-polygon",
"params": {
"pointsSource": {
"type": "source",
"params": {
"query": "select * from analysis_rent_listings"
}
},
"polygonsSource": {
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from analysis_banks"
}
},
"radius": 250
}
}
}
}),
cartocss: cartocss('green', 1.0)
}
},
{
type: 'cartodb',
options: {
sql: "select * from analysis_rent_listings",
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
}
])
},
{
desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
mapConfig: mapConfig([
{
type: 'cartodb',
options: {
sql: "select * from analysis_rent_listings",
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
},
{
type: 'analysis',
options: {
def: analysisDef({
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from analysis_banks"
}
},
"radius": 300
}
}),
cartocss: cartocss('magenta', 0.5)
}
},
{
type: 'analysis',
options: {
def: analysisDef({
"type": "point-in-polygon",
"params": {
"pointsSource": {
"type": "source",
"params": {
"query": "select * from analysis_rent_listings"
}
},
"polygonsSource": {
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from analysis_banks"
}
},
"radius": 300
}
}
}
}),
cartocss: cartocss('green', 1.0)
}
}
])
},
{
skip: true,
desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
mapConfig: mapConfig([
{
type: 'cartodb',
options: {
"source": { id: "a" },
"cartocss": DEFAULT_MULTITYPE_STYLE,
"cartocss_version": "2.3.0"
}
},
{
type: 'cartodb',
options: {
"source": { id: "b1" },
"cartocss": cartocss('green', 1.0),
"cartocss_version": "2.3.0"
}
},
{
type: 'cartodb',
options: {
"source": { id: "b2" },
"cartocss": cartocss('magenta', 0.5),
"cartocss_version": "2.3.0"
}
}
],
[
{
id: "b2",
options: {
def: analysisDef({
"type": "count-in-polygon",
"id": "a0",
"params": {
"columnName": 'count_airbnb',
"pointsSource": {
"type": "source",
"params": {
query: "select * from analysis_rent_listings"
},
dataviews: {
price_histogram: {
type: 'histogram',
options: {
column: 'price'
}
}
}
},
"polygonsSource": {
"id": "b1",
"type": "buffer",
"params": {
"source": {
"id": "b0",
"type": "source",
"params": {
query: "select * from analysis_banks"
}
},
"radius": 250
},
dataviews: {
bank_category: {
type: 'aggregation',
options: {
column: 'bank'
}
}
}
}
},
dataviews: {
count_histogram: {
type: 'histogram',
options: {
column: 'count_airbnb'
}
}
}
}),
cartocss: cartocss('green', 1.0)
}
}
])
},
{
skip: true,
desc: 'I. Distribution centers',
mapConfig: mapConfig(
// layers
[
{
type: 'cartodb',
options: {
"source": { id: "b0" },
"cartocss": [
"#distribution_centers {",
" marker-fill-opacity: 1.0;",
" marker-line-color: #FFF;",
" marker-line-width: 0.5;",
" marker-line-opacity: 0.7;",
" marker-placement: point;",
" marker-type: ellipse;",
" marker-width: 8;",
" marker-fill: blue;",
" marker-allow-overlap: true;",
"}"
].join('\n'),
"cartocss_version": "2.3.0"
source: {
id: 'a1'
},
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
},
{
type: 'cartodb',
options: {
"source": { id: "a0" },
"cartocss": [
"#shops {",
" marker-fill-opacity: 1.0;",
" marker-line-color: #FFF;",
" marker-line-width: 0.5;",
" marker-line-opacity: 0.7;",
" marker-placement: point;",
" marker-type: ellipse;",
" marker-width: 8;",
" marker-fill: red;",
" marker-allow-overlap: true;",
"}"
].join('\n'),
"cartocss_version": "2.3.0"
}
},
{
type: 'cartodb',
options: {
"source": { id: "a1" },
"cartocss": [
"#routing {",
" line-color: ramp([routing_time], colorbrewer(Reds));",
" line-width: ramp([routing_time], 2, 8);",
" line-opacity: 1.0;",
"}"
].join('\n'),
"cartocss_version": "2.3.0"
sql: 'select * from analysis_rent_listings',
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
}
],
// dataviews
{
distribution_center_name_category: {
source: { id: 'b0' },
type: 'aggregation',
options: {
column: 'name'
}
},
time_histogram: {
source: { id: 'a1' },
type: 'histogram',
options: {
column: 'routing_time'
}
},
distance_histogram: {
source: { id: 'a1' },
type: 'histogram',
options: {
column: 'routing_distance'
}
}
},
// analysis
{},
[
{
id: 'a1',
type: 'routing-n-to-n',
params: {
// distanceColumn: 'routing_distance',
// timeColumn: 'routing_time',
originSource: {
id: 'b0',
type: 'source',
params: {
query: 'select * from distribution_centers'
}
},
destinationSource: {
id: 'a0',
type: 'source',
params: {
query: 'select * from shops'
}
}
}
}
pointInPolygonDef
]
)
},
{
skip: true,
desc: 'II. Population analysis',
desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
mapConfig: mapConfig(
// layers
[
{
type: 'cartodb',
options: {
"source": { id: "a2" },
"cartocss": [
"#count_in_polygon {",
" polygon-opacity: 1.0",
" line-color: #FFF;",
" line-width: 0.5;",
" line-opacity: 0.7",
" polygon-fill: ramp([estimated_people], colorbrewer(Reds));",
"}"
].join('\n'),
"cartocss_version": "2.3.0"
}
},
{
type: 'cartodb',
options: {
"source": { id: "a0" },
"cartocss": DEFAULT_MULTITYPE_STYLE,
"cartocss_version": "2.3.0"
}
}
],
// dataviews
{
total_population_formula: {
"source": { id: "a3" },
type: 'formula',
options: {
column: 'total_population',
operation: 'sum'
}
},
people_histogram: { // this injects a range filter at `a2` node output
"source": { id: "a2" },
type: 'histogram',
options: {
column: 'estimated_people'
}
},
subway_line_category: { // this injects a category filter at `a0` node output
"source": { id: "a0" },
type: 'aggregation',
options: {
column: 'subway_line'
}
}
},
// analysis
[
{
id: 'a3',
// this will union the polygons, produce just one polygon, and calculate the total population for it
type: 'total-population',
params: {
columnName: 'total_population',
source: {
id: 'a2',
type: 'estimated-population',
params: {
columnName: 'estimated_people',
source: {
id: 'a1',
type: 'trade-area',
params: {
source: {
"id": "a0",
"type": "source",
"params": {
query: "select * from subway_stops"
}
},
kind: 'walk',
time: 300
}
}
}
}
}
}
])
},
{
skip: true,
desc: 'III. Point in polygon',
mapConfig: mapConfig(
// layers
[
{
type: 'cartodb',
options: {
"source": { id: "a1" },
"cartocss": [
"#count_in_polygon {",
" polygon-opacity: 1.0",
" line-color: #FFF;",
" line-width: 0.5;",
" line-opacity: 0.7",
" polygon-fill: ramp([count_people], colorbrewer(Reds));",
"}"
].join('\n'),
"cartocss_version": "2.3.0"
sql: 'select * from analysis_rent_listings',
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
},
{
type: 'cartodb',
options: {
source: {
id: 'a1'
},
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
},
{
type: 'cartodb',
options: {
source: {
id: 'b1'
},
cartocss: DEFAULT_MULTITYPE_STYLE,
cartocss_version: '2.3.0'
}
}
],
// dataviews
{
age_histogram: {
"source": { id: "a0" },
type: 'histogram',
options: {
column: 'age'
}
},
income_histogram: {
"source": { id: "a0" },
type: 'histogram',
options: {
column: 'income'
}
},
gender_category: {
"source": { id: "a0" },
type: 'aggregation',
options: {
column: 'gender'
}
}
},
// analysis
{},
[
{
"id": "a1",
"type": "count-in-polygon",
"params": {
"columnName": 'count_people',
"pointsSource": {
"id": 'a0',
"type": "source",
"params": {
query: "select the_geom, age, gender, income from people"
}
},
"polygonsSource": {
"id": "b0",
"type": "source",
"params": {
query: "select * from postal_codes"
}
}
}
}
bufferDef,
pointInPolygonDef
]
)
}
];
useCases.forEach(function(useCase, imageIdx) {
useCases.forEach(function (useCase) {
if (!!useCase.skip) {
debug(JSON.stringify(useCase.mapConfig, null, 4));
return debug(JSON.stringify(useCase.mapConfig, null, 4));
}
it.skip('should implement use case: "' + useCase.desc + '"', function(done) {
it(`should implement use case: '${useCase.desc}'`, function (done) {
var testClient = new TestClient(useCase.mapConfig, 1234);
var tile = useCase.tile || TILE_ANALYSIS_TABLES;
testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) {
testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) {
assert.ok(!err, err);
image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
//image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
assert.equal(image.width(), 256);

65
test/acceptance/analysis/error-cases.js
View File

@ -373,5 +373,70 @@ describe('analysis-layers error cases', function() {
});
});
it('should return "function does not exist" indicating the node_id and context', function(done) {
var mapConfig = createMapConfig([{
"type": "cartodb",
"options": {
"source": {
"id": "HEAD"
},
"cartocss": '#polygons { polygon-fill: red; }',
"cartocss_version": "2.3.0"
}
}], {}, [{
"id": "HEAD",
"type": "buffer",
"params": {
"source": {
"id": "HEAD2",
"type": "buffer",
"params": {
"source": {
"id": "HEAD3",
"type": 'deprecated-sql-function',
"params": {
"id": "HEAD4",
"function_name": 'DEP_EXT_does_not_exist_fn',
"primary_source": {
"type": 'source',
"params": {
"query": "select * from populated_places_simple_reduced"
}
},
"function_args": ['wadus']
}
},
"radius": 10
}
},
"radius": 10
}
}]);
var testClient = new TestClient(mapConfig, 1234);
testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
assert.ok(!err, err);
assert.equal(layergroupResult.errors.length, 1);
assert.equal(
layergroupResult.errors[0],
'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist'
);
assert.equal(layergroupResult.errors_with_context[0].type, 'analysis');
assert.equal(
layergroupResult.errors_with_context[0].message,
'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist'
);
assert.equal(layergroupResult.errors_with_context[0].analysis.id, 'HEAD');
assert.equal(layergroupResult.errors_with_context[0].analysis.type, 'buffer');
assert.equal(layergroupResult.errors_with_context[0].analysis.node_id, 'HEAD3');
testClient.drain(done);
});
});
});

441
test/acceptance/buffer-size-format.js Normal file
View File

@ -0,0 +1,441 @@
require('../support/test_helper');
var fs = require('fs');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
var mapnik = require('windshaft').mapnik;
var IMAGE_TOLERANCE_PER_MIL = 5;
var CARTOCSS_LABELS = [
'#layer {',
' polygon-fill: #374C70;',
' polygon-opacity: 0.9;',
' line-width: 1;',
' line-color: #FFF;',
' line-opacity: 0.5;',
'}',
'#layer::labels {',
' text-name: [name];',
' text-face-name: \'DejaVu Sans Book\';',
' text-size: 20;',
' text-fill: #FFFFFF;',
' text-label-position-tolerance: 0;',
' text-halo-radius: 1;',
' text-halo-fill: #6F808D;',
' text-dy: -10;',
' text-allow-overlap: true;',
' text-placement: point;',
' text-placement-type: dummy;',
'}'
].join('\n');
function createMapConfig (bufferSize, cartocss) {
cartocss = cartocss || CARTOCSS_LABELS;
return {
version: '1.6.0',
buffersize: bufferSize,
layers: [{
type: "cartodb",
options: {
sql: [
'select',
' *',
'from',
' populated_places_simple_reduced',
].join('\n'),
cartocss: cartocss,
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
}]
};
}
describe('buffer size per format', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
mapConfig: createMapConfig({ png: 0, 'grid.json': 0 }),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
mapConfig: createMapConfig({ png: 128, 'grid.json': 128 }),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get mvt tile using buffer-size 0',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.mvt',
mapConfig: createMapConfig({ mvt: 0 }),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
assert.equal(features.length, 1);
callback();
}
},
{
desc: 'should get mvt tile using buffer-size 128',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.mvt',
mapConfig: createMapConfig({ mvt: 128 }),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
assert.equal(features.length, 9);
callback();
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
mapConfig: createMapConfig({ 'grid.json': 0 }),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
mapConfig: createMapConfig({ 'grid.json': 128 }),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
}
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.mapConfig, 1234);
var coords = test.coords;
var options = {
format: test.format,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
// tile.save(test.fixturePath);
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});
function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) {
cartocss = cartocss || CARTOCSS_LABELS;
return {
"version": "0.0.1",
"name": name,
"placeholders": placeholders || {
"buffersize": {
"type": "number",
"default": 0
}
},
"layergroup": createMapConfig(buffersize)
};
}
describe('buffer size per format for named maps', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0 (default value in template)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 (placehoder value)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-custom-buffer-size', { png: '<%= buffersize %>'}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 0 (default value in template by format)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize_png: 0 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-default-buffer-size-by-format', {
png: '<%= buffersize_png %>'
}, {
"buffersize_png": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize_png: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-custom-buffer-size-by-format', {
png: '<%= buffersize_png %>'
}, {
"buffersize_png": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: { buffersize_gridjson: 0 },
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', {
'grid.json': '<%= buffersize_gridjson %>'
}, {
"buffersize_gridjson": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: { buffersize_gridjson: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', {
'grid.json': '<%= buffersize_gridjson %>'
}, {
"buffersize_gridjson": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
}
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png');
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});
describe('buffer size per format for named maps w/o placeholders', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-no-buffer-size-png-0', {}, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-no-buffer-size-png-128', {}, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get mvt tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
placeholders: {
buffersize: {
mvt: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt',
template: createBufferSizeTemplate('named-no-buffer-size-mvt', {}, {}),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
var dataFixture = fs.readFileSync(this.fixturePath);
var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
vtile.setDataSync(dataFixture);
var vtileJSON = vtile.toJSON();
var vtileFeatures = vtileJSON[0].features;
assert.equal(features.length, vtileFeatures.length);
callback();
}
},
{
desc: 'should get mvt tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
placeholders: {
buffersize: {
mvt: 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt',
template: createBufferSizeTemplate('named-no-buffer-size-mvt-128', {}, {}),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
var dataFixture = fs.readFileSync(this.fixturePath);
var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
vtile.setDataSync(dataFixture);
var vtileJSON = vtile.toJSON();
var vtileFeatures = vtileJSON[0].features;
assert.equal(features.length, vtileFeatures.length);
callback();
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: {
buffersize: {
'grid.json': 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
template: createBufferSizeTemplate('named-no-buffer-size-grid-json-0', {}, {}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: {
buffersize: {
'grid.json': 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
template: createBufferSizeTemplate('named-no-buffer-size-grid-json-128', {}, {}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
},
{
desc: 'should get png tile using buffer-size 0' +
' overriden by template params with no buffersize in mapconfig',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-no-buffer-size-mapconfig-png-0', undefined, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save(test.fixturePath);
// require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
// require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});

393
test/acceptance/cache/cache_headers.js vendored Normal file
View File

@ -0,0 +1,393 @@
var testHelper = require('../../support/test_helper');
var assert = require('../../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../../lib/cartodb/server');
var serverOptions = require('../../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var LayergroupToken = require('../../support/layergroup-token');
describe('get requests with cache headers', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var statusOkResponse = {
status: 200
};
var mapConfigs = [
{
"description": "cache headers should be present",
"cache_headers": {
"x_cache_channel": {
"db_name": "test_windshaft_cartodb_user_1_db",
"tables": ["public.test_table"]
},
"surrogate_keys": "t:77pJnX"
},
"data":
{
version: '1.5.0',
layers: [
{
options: {
source: {
id: "2570e105-7b37-40d2-bdf4-1af889598745"
},
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
],
analyses: [
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "source",
"params": {
"query": "select * from test_table limit 2"
}
}
]
},
},
{
"description": "cache headers should be present and be composed with source table name",
"cache_headers": {
"x_cache_channel": {
"db_name": "test_windshaft_cartodb_user_1_db",
"tables": ["public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d",
"public.test_table"]
},
"surrogate_keys": "t:77pJnX t:iL4eth"
},
"data":
{
version: '1.5.0',
layers: [
{
options: {
source: {
id: "2570e105-7b37-40d2-bdf4-1af889598745"
},
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
],
analyses: [
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from test_table limit 2"
}
},
"radius": 50000
}
}
]
}
}];
var layergroupRequest = function(mapConfig) {
return {
url: '/api/v1/map?api_key=1234&config=' + encodeURIComponent(JSON.stringify(mapConfig)),
method: 'GET',
headers: {
host: 'localhost'
}
};
};
function getRequest(url, addApiKey, callbackName) {
var params = {};
if (!!addApiKey) {
params.api_key = '1234';
}
if (!!callbackName) {
params.callback = callbackName;
}
return {
url: url + '?' + qs.stringify(params),
method: 'GET',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
};
}
function validateCacheHeaders(done, expectedCacheHeaders) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(res.headers['x-cache-channel']);
assert.ok(res.headers['surrogate-key']);
if (expectedCacheHeaders) {
validateXChannelHeaders(res.headers, expectedCacheHeaders);
assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys);
}
done();
};
}
function validateXChannelHeaders(headers, expectedCacheHeaders) {
var dbName = headers['x-cache-channel'].split(':')[0];
var tables = headers['x-cache-channel'].split(':')[1].split(',').sort();
assert.equal(dbName, expectedCacheHeaders.x_cache_channel.db_name);
assert.deepEqual(tables, expectedCacheHeaders.x_cache_channel.tables.sort());
}
function noCacheHeaders(done) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(
!res.headers['x-cache-channel'],
'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
);
assert.ok(
!res.headers['surrogate-key'],
'did not expect surrogate-key header, got: `' + res.headers['surrogate-key'] + '`'
);
done();
};
}
function withLayergroupId(mapConfig, callback) {
assert.response(
server,
layergroupRequest(mapConfig),
statusOkResponse,
function(res, err) {
if (err) {
return callback(err);
}
var layergroupId = JSON.parse(res.body).layergroupid;
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
callback(null, layergroupId, res);
}
);
}
mapConfigs.forEach(function(mapConfigData) {
describe(mapConfigData.description, function() {
var mapConfig = mapConfigData.data;
var expectedCacheHeaders = mapConfigData.cache_headers;
it('/api/v1/map Map instantiation', function(done) {
var testFn = validateCacheHeaders(done, expectedCacheHeaders);
withLayergroupId(mapConfig, function(err, layergroupId, res) {
testFn(res);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
});
});
describe('cache headers should NOT be present', function() {
it('/', function(done) {
assert.response(
server,
getRequest('/'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/version', function(done) {
assert.response(
server,
getRequest('/version'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/health', function(done) {
assert.response(
server,
getRequest('/health'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/api/v1/map/named list named maps', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named', true),
statusOkResponse,
noCacheHeaders(done)
);
});
describe('with named maps', function() {
var templateName = 'x_cache';
beforeEach(function(done) {
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
layergroup: mapConfigs[0].data
};
var namedMapRequest = {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(template)
};
assert.response(
server,
namedMapRequest,
statusOkResponse,
function(res, err) {
done(err);
}
);
});
afterEach(function(done) {
assert.response(
server,
{
url: '/api/v1/map/named/' + templateName + '?api_key=1234',
method: 'DELETE',
headers: {
host: 'localhost'
}
},
{
status: 204
},
function(res, err) {
done(err);
}
);
});
it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
statusOkResponse,
noCacheHeaders(done)
);
});
});
});
});

178
test/acceptance/dataviews/aggregation.js
View File

@ -145,4 +145,182 @@ describe('aggregations happy cases', function() {
});
});
});
var widgetSearchExpects = {
'count': [ { category: 'other_a', value: 3 } ],
'sum': [ { category: 'other_a', value: 6 } ],
'avg': [ { category: 'other_a', value: 2 } ],
'max': [ { category: 'other_a', value: 3 } ],
'min': [ { category: 'other_a', value: 1 } ]
};
Object.keys(operations_and_values).forEach(function (operation) {
var description = 'should search OTHER category using "' + operation + '"';
it(description, function (done) {
this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) {
assert.ifError(err);
assert.ok(searchResult);
assert.equal(searchResult.type, 'aggregation');
assert.equal(searchResult.categories.length, 1);
assert.deepEqual(
searchResult.categories,
widgetSearchExpects[operation]
);
done();
});
});
});
});
describe('aggregation-dataview: special float values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
val_aggregation: {
source: {
id: 'a0'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'avg',
aggregationColumn: 'val'
}
},
sum_aggregation_numeric: {
source: {
id: 'a1'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'sum',
aggregationColumn: 'val'
}
}
},
[
{
"id": "a0",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS val,',
' CASE',
' WHEN x % 2 = 0 THEN \'category_1\'',
' ELSE \'category_2\'',
' END AS cat',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}, {
"id": "a1",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 3 = 0 THEN \'NaN\'::numeric',
' WHEN x % 3 = 1 THEN x',
' ELSE x',
' END AS val,',
' CASE',
' WHEN x % 2 = 0 THEN \'category_1\'',
' ELSE \'category_2\'',
' END AS cat',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}
]
);
// Source a0
// -----------------------------------------------
// the_geom_webmercator | val | cat
// ----------------------+-----------+------------
// | -Infinity | category_2
// | NaN | category_1
// | 3 | category_2
// | Infinity | category_1
// | -Infinity | category_2
// | NaN | category_1
// | 7 | category_2
// | Infinity | category_1
// | -Infinity | category_2
// | NaN | category_1
// | 11 | category_2
// | " | "
var filters = [{ own_filter: 0 }, {}];
filters.forEach(function (filter) {
it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) {
assert.ifError(err);
assert.ok(dataview.infinities === (250 + 250));
assert.ok(dataview.nans === 250);
assert.ok(dataview.categories.length === 1);
dataview.categories.forEach(function (category) {
assert.ok(category.category === 'category_2');
assert.ok(category.value === 501);
});
done();
});
});
it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) {
assert.ifError(err);
assert.ok(dataview.nans === 333);
assert.ok(dataview.categories.length === 2);
dataview.categories.forEach(function (category) {
assert.ok(category.value !== null);
});
done();
});
});
});
});

80
test/acceptance/dataviews/formula.js Normal file
View File

@ -0,0 +1,80 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
describe('formula-dataview: special float values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
val_formula: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: 'val',
operation: 'avg'
}
}
},
[
{
"id": "a0",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS val',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}
]
);
it('should filter infinities out and count them in the summary', function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('val_formula', {}, function(err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.result, 501);
assert.ok(dataview.infinities === (250 + 250));
assert.ok(dataview.nans === 250);
done();
});
});
});

1112
test/acceptance/dataviews/histogram.js
View File

File diff suppressed because it is too large Load Diff

221
test/acceptance/dataviews/overviews.js
View File

@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() {
params: {
query: 'select * from test_table_overviews'
}
},
{
id: 'data-source-special-float-values',
type: 'source',
params: {
query: 'select * from test_special_float_values_table_overviews'
}
}
],
dataviews: {
@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() {
aggregationColumn: 'name',
}
},
test_categories_special_values: {
type: 'aggregation',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'name',
aggregation: 'sum',
aggregationColumn: 'value',
}
},
test_histogram: {
type: 'histogram',
source: {id: 'data-source'},
@ -160,6 +178,16 @@ describe('dataviews using tables with overviews', function() {
bins: 2
}
},
test_histogram_special_values: {
type: 'histogram',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'value',
bins: 2
}
},
test_avg: {
type: 'formula',
source: {id: 'data-source'},
@ -168,6 +196,16 @@ describe('dataviews using tables with overviews', function() {
operation: 'avg'
}
},
test_formula_sum_special_values: {
type: 'formula',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'value',
operation: 'sum'
}
},
test_count: {
type: 'formula',
source: {id: 'data-source'},
@ -202,6 +240,17 @@ describe('dataviews using tables with overviews', function() {
cartocss_version: '2.3.0',
source: { id: 'data-source' }
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_special_float_values_table_overviews',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
source: {
id: 'data-source-special-float-values'
}
}
}
]
};
@ -212,7 +261,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":15,
"infinities": 0,
"nans": 0,
"nulls":0,
"type":"formula"
});
testClient.drain(done);
});
@ -224,7 +280,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"avg",
"result":3,
"nulls":0,
"type":"formula",
"infinities": 0,
"nans": 0
});
testClient.drain(done);
});
@ -236,7 +299,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"count",
"result":5,
"nulls":0,
"type":"formula",
"infinities": 0,
"nans": 0
});
testClient.drain(done);
});
@ -248,7 +318,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "max",
"result": 5,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@ -260,7 +337,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "min",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@ -275,7 +359,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":15,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
@ -372,7 +463,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
});
@ -383,7 +481,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"avg",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
@ -395,7 +500,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"count",
"result":1,
"infinities": 0,
"nans": 0,
"nulls":0,
"type":"formula"
});
testClient.drain(done);
});
@ -407,7 +519,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "max",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@ -419,7 +538,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "min",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@ -437,7 +563,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
});
@ -445,5 +578,69 @@ describe('dataviews using tables with overviews', function() {
});
describe('aggregation special float values', function () {
var params = {};
it("should expose an aggregation dataview filtering special float values out", function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_categories_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
aggregation: 'sum',
count: 5,
nulls: 0,
nans: 1,
infinities: 1,
min: 6,
max: 6,
categoriesCount: 1,
categories: [ { category: 'Hawai', value: 6, agg: false } ],
type: 'aggregation'
});
testClient.drain(done);
});
});
it('should expose a histogram dataview filtering special float values out', function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_histogram_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
bin_width: 0,
bins_count: 1,
bins_start: 3,
nulls: 0,
infinities: 1,
nans: 1,
avg: 3,
bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ],
type: 'histogram'
});
testClient.drain(done);
});
});
it('should expose a formula (sum) dataview filtering special float values out', function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
operation: 'sum',
result: 6,
nulls: 0,
nans: 1,
infinities: 1,
type: 'formula'
});
testClient.drain(done);
});
});
});
});
});

311
test/acceptance/limits.js
View File

@ -1,311 +0,0 @@
var testHelper = require('../support/test_helper');
var assert = require('../support/assert');
var _ = require('underscore');
var redis = require('redis');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var LayergroupToken = require('../support/layergroup-token');
describe('render limits', function() {
var layergroupUrl = '/api/v1/map';
var redisClient = redis.createClient(global.environment.redis.port);
var server;
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var user = 'localhost';
var pointSleepSql = "SELECT pg_sleep(0.5)," +
" 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id";
var pointCartoCss = '#layer { marker-fill:red; }';
var polygonSleepSql = "SELECT pg_sleep(0.5)," +
" ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id";
var polygonCartoCss = '#layer { polygon-fill:red; }';
function singleLayergroupConfig(sql, cartocss) {
return {
version: '1.0.0',
layers: [
{
type: 'mapnik',
options: {
sql: sql,
cartocss: cartocss,
cartocss_version: '2.0.1'
}
}
]
};
}
function createRequest(layergroup, userHost) {
return {
url: layergroupUrl,
method: 'POST',
headers: {
host: userHost,
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
};
}
function withRenderLimit(user, renderLimit, callback) {
redisClient.SELECT(5, function(err) {
if (err) {
return callback(err);
}
var userLimitsKey = 'limits:tiler:' + user;
redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) {
if (err) {
return callback(err);
}
keysToDelete[userLimitsKey] = 5;
return callback();
});
});
}
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategyEnabled;
before(function() {
onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
});
after(function() {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled;
});
it("layergroup creation fails if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed.errors, [ 'Render timed out' ]);
done();
}
);
});
});
it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done();
}
);
});
});
it("layergroup creation works if test tile is fast but tile request fails if they are slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed.errors, ['Render timed out']);
done();
}
);
}
);
});
});
it("tile request does not fail if user limit is high enough", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
done(err);
}
);
}
);
});
});
});
describe('with onTileErrorStrategy', function() {
it("layergroup creation works even if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done();
}
);
});
});
it("layergroup creation and tile requests works even if they are slow but returns fallback", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
if (err) {
done(err);
}
var referenceImagePath = './test/fixtures/render-timeout-fallback.png';
assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25,
function(imgErr/*, similarity*/) {
done(imgErr);
}
);
}
);
}
);
});
});
});
});

63
test/acceptance/mvt.js Normal file
View File

@ -0,0 +1,63 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
function createMapConfig (sql = TestClient.SQL.ONE_POINT) {
return {
version: '1.6.0',
layers: [{
type: "cartodb",
options: {
sql: sql,
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
}]
};
}
describe('mvt', function () {
const testCases = [
{
desc: 'should get empty mvt with code 204 (no content)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
response: {
status: 204,
headers: {
'Content-Type': undefined
}
},
mapConfig: createMapConfig(TestClient.SQL.EMPTY)
},
{
desc: 'should get mvt tile with code 200 (ok)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
response: {
status: 200,
headers: {
'Content-Type': 'application/x-protobuf'
}
},
mapConfig: createMapConfig()
}
];
testCases.forEach(function (test) {
it(test.desc, done => {
const testClient = new TestClient(test.mapConfig, 1234);
const { z, x, y } = test.coords;
const { format, response } = test;
testClient.getTile(z, x, y, { format, response }, (err, res) => {
assert.ifError(err);
assert.equal(res.statusCode, test.response.status);
testClient.drain(done);
});
});
});
});

69
test/acceptance/named_maps_static_view.js
View File

@ -21,7 +21,7 @@ describe('named maps static view', function() {
var IMAGE_TOLERANCE = 20;
function createTemplate(view) {
function createTemplate(view, layers) {
return {
version: '0.0.1',
name: templateName,
@ -36,7 +36,7 @@ describe('named maps static view', function() {
},
view: view,
layergroup: {
layers: [
layers: layers || [
{
type: 'mapnik',
options: {
@ -192,10 +192,73 @@ describe('named maps static view', function() {
}
getStaticMap({ zoom: 3 }, function(err, img) {
assert.ok(!err);
img.save('/tmp/static.png');
assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done);
});
});
});
it('should return override bbox', function (done) {
var view = {
bounds: {
west: 0,
south: 0,
east: 45,
north: 45
},
zoom: 4,
center: {
lng: 40,
lat: 20
}
};
templateMaps.addTemplate(username, createTemplate(view), function (err) {
if (err) {
return done(err);
}
getStaticMap({ bbox: '0,45,90,45' }, function(err, img) {
assert.ok(!err);
assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), IMAGE_TOLERANCE, done);
});
});
});
it('should allow to select the layers to render', function (done) {
var view = {
bounds: {
west: 0,
south: 0,
east: 45,
north: 45
}
};
var layers = [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: <%= color %>; }',
cartocss_version: '2.3.0'
}
},
{
type: 'mapnik',
options: {
sql: 'select ST_Transform(ST_MakeEnvelope(-45, -45, 45, 45, 4326), 3857) the_geom_webmercator',
cartocss: '#layer { polygon-fill: <%= color %>; }',
cartocss_version: '2.3.0'
}
}
];
templateMaps.addTemplate(username, createTemplate(view, layers), function (err) {
if (err) {
return done(err);
}
getStaticMap({ layer: 0 }, function(err, img) {
assert.ok(!err);
assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
});
});
});
});

37
test/acceptance/ported/multilayer_error_cases.js
View File

@ -5,6 +5,7 @@ var step = require('step');
var cartodbServer = require('../../../lib/cartodb/server');
var ServerOptions = require('./support/ported_server_options');
var testClient = require('./support/test_client');
var TestClient = require('../../support/test-client');
var BaseController = require('../../../lib/cartodb/controllers/base');
@ -23,6 +24,14 @@ describe('multilayer error cases', function() {
BaseController.prototype.req2params = req2paramsFn;
});
// var client = null;
afterEach(function(done) {
if (this.client) {
return this.client.drain(done);
}
return done();
});
it("post layergroup with wrong Content-Type", function(done) {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
@ -153,24 +162,16 @@ describe('multilayer error cases', function() {
]
};
ServerOptions.afterLayergroupCreateCalls = 0;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
var parsed = JSON.parse(res.body);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
var error = parsed.errors[0];
assert.ok(error.match(/column "missing" does not exist/m), error);
// TODO: check which layer introduced the problem ?
done();
} catch (err) { done(err); }
this.client = new TestClient(layergroup);
this.client.getLayergroup({status: 400}, function(err, parsed) {
assert.ok(!err, err);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
var error = parsed.errors[0];
assert.ok(error.match(/column "missing" does not exist/m), error);
done();
});
});

4
test/acceptance/ported/server_png8_format.js
View File

@ -16,13 +16,13 @@ describe('server_png8_format', function() {
var serverOptionsPng32 = ServerOptions;
serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore);
serverOptionsPng32.grainstore.mapnik_tile_format = 'png32';
var serverPng32 = new cartodbServer(serverOptionsPng32);
var serverPng32 = cartodbServer(serverOptionsPng32);
serverPng32.setMaxListeners(0);
var serverOptionsPng8 = ServerOptions;
serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore);
serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h';
var serverPng8 = new cartodbServer(serverOptionsPng8);
var serverPng8 = cartodbServer(serverOptionsPng8);
serverPng8.setMaxListeners(0);

71
test/acceptance/special-numeric-values.js Normal file
View File

@ -0,0 +1,71 @@
require('../support/test_helper');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
describe('special numeric values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var ATTRIBUTES_LAYER = 1;
function createMapConfig(sql, id, columns) {
return {
version: '1.6.0',
layers: [
{
type: 'mapnik',
options: {
sql: "select 1 as id, 'SRID=4326;POINT(0 0)'::geometry as the_geom",
cartocss: '#style { }',
cartocss_version: '2.0.1'
}
},
{
type: 'mapnik',
options: {
sql: sql || "select 1 as i, 6 as n, 'SRID=4326;POINT(0 0)'::geometry as the_geom",
attributes: {
id: id || 'i',
columns: columns || ['n']
},
cartocss: '#style { }',
cartocss_version: '2.0.1'
}
}
]
};
}
it('should retrieve special numeric values', function (done) {
var featureId = 1;
var sql = [
'SELECT',
' 1 as cartodb_id,',
' null::geometry the_geom_webmercator,',
' \'infinity\'::float as infinity,',
' \'-infinity\'::float as _infinity,',
' \'NaN\'::float as nan'
].join('\n');
var id = 'cartodb_id';
var columns = ['infinity', '_infinity', 'nan'];
var mapConfig = createMapConfig(sql, id, columns);
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getFeatureAttributes(featureId, ATTRIBUTES_LAYER, {}, function (err, attributes) {
assert.ifError(err);
assert.equal(attributes.infinity, 'Infinity');
assert.equal(attributes._infinity, '-Infinity');
assert.equal(attributes.nan, 'NaN');
done();
});
});
});

279
test/acceptance/stats/mapnik_stats_layergroup.js Normal file
View File

@ -0,0 +1,279 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('Create mapnik layergroup', function() {
before(function() {
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerStats = true;
});
after(function() {
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var mapnikLayer1 = {
type: 'mapnik',
options: {
sql: 'select * from test_table limit 1',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer2 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_2 limit 2',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer3 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_3 limit 3',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer4 = {
type: 'mapnik',
options: {
sql: [
'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
' from test_table t1, test_table_2 t2',
' where t1.cartodb_id = t2.cartodb_id;'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var mapnikLayerGeomColumn = {
type: 'mapnik',
options: {
sql: 'select *, the_geom as my_geom from test_table_3 limit 2',
geom_column: 'my_geom',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId(index) {
return 'layer' + index;
}
function typeLayerId(type, index) {
return type + '-' + mapnikBasicLayerId(index);
}
it('with one mapnik layer should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
testClient.drain(done);
});
});
it('with two mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
testClient.drain(done);
});
});
it('with three mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2,
mapnikLayer3
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2));
assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3);
testClient.drain(done);
});
});
it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer3,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
assert.ok(!layergroup.metadata.layers[2]);
testClient.drain(done);
});
});
it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].type, 'mapnik');
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[1].type, 'http');
testClient.drain(done);
});
});
it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss);
testClient.drain(done);
});
});
it('should work with different geom_column', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayerGeomColumn
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
// we don't care about stats here as is an aliased column
assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount'));
testClient.drain(done);
});
});
it('should not include the stats part if the FF is disabled', function(done) {
global.environment.enabledFeatures.layerStats = false;
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats'));
assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
assert.equal(layergroup.metadata.layers[2].type, 'http');
testClient.drain(done);
});
});
});

225
test/acceptance/stats/multilayer_stats.js Normal file
View File

@ -0,0 +1,225 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('multilayer stats disabled', function() {
before(function () {
this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata;
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerMetadata = true;
global.environment.enabledFeatures.layerStats = false;
});
after(function () {
global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig;
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
function testLayerMetadataStats(testScenario) {
it(testScenario.desc, function(done) {
var mapConfig = {
version: '1.3.0',
layers: testScenario.layers
};
var testClient = new TestClient(mapConfig);
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
layergroup.metadata.layers.forEach(function (layer) {
if (layer.type !== 'torque' && layer.type !== 'mapnik') {
assert.ok(!('stats' in layer.meta));
} else if (layer.type !== 'torque') {
assert.ok(!('stats' in layer.meta));
assert.ok('cartocss' in layer.meta);
} else {
assert.ok('cartocss' in layer.meta);
// check torque metadata at least match in number
var torqueLayers = mapConfig.layers.filter(function(layer) { return layer.type === 'torque'; });
if (torqueLayers.length) {
assert.equal(Object.keys(layergroup.metadata.torque).length, torqueLayers.length);
}
}
});
testClient.drain(done);
});
});
}
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var torqueLayer = {
type: 'torque',
options: {
sql: "select 1 id, '1970-01-02'::date d, 'POINT(0 0)'::geometry the_geom_webmercator",
cartocss: [
"Map {",
"-torque-frame-count:2;",
"-torque-resolution:3;",
"-torque-time-attribute:d;",
"-torque-aggregation-function:'count(id)';",
"}"
].join(' '),
cartocss_version: '2.0.1'
}
};
var mapnikLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikInteractivityLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var cartodbInteractivityLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbWadusInteractivityLayer = {
type: 'cartodb',
options: {
sql: sqlWadus,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'wadus'
}
};
var noTypeLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var noTypeInteractivityLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var testScenarios = [
{
desc: 'one layer, no interactivity',
layers: [cartodbLayer]
},
{
desc: 'two layers, different interactivity columns',
layers: [
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'torque + interactivity layers',
layers: [
torqueLayer,
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'interactivity + torque + interactivity',
layers: [
cartodbInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'http + interactivity + torque + no interactivity + torque + interactivity',
layers: [
httpLayer,
cartodbInteractivityLayer,
torqueLayer,
cartodbLayer,
torqueLayer,
cartodbWadusInteractivityLayer
]
},
{
desc: 'mapnik type – two layers, interactivity mix',
layers: [
mapnikLayer,
mapnikInteractivityLayer
]
},
{
desc: 'mapnik type – http + interactivity + torque + interactivity',
layers: [
httpLayer,
mapnikInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'no type – two layers, interactivity mix',
layers: [
noTypeLayer,
noTypeInteractivityLayer
]
},
{
desc: 'no type – http + interactivity + torque + interactivity',
layers: [
httpLayer,
noTypeInteractivityLayer,
torqueLayer,
noTypeInteractivityLayer
]
}
];
testScenarios.forEach(testLayerMetadataStats);
});

6
test/acceptance/templates.js
View File

@ -1052,8 +1052,9 @@ describe('template_api', function() {
'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
var cc = res.headers['x-cache-channel'];
var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
assert.ok(cc);
assert.ok(cc.match, /ciao/, cc);
assert.equal(cc, expectedCC);
// hack simulating restart...
server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache
var get_request = {
@ -1072,8 +1073,9 @@ describe('template_api', function() {
'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
var cc = res.headers['x-cache-channel'];
var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart");
assert.ok(cc.match, /ciao/, cc);
assert.equal(cc, expectedCC);
return null;
},
function deleteTemplate(err)

786
test/acceptance/user-database-timeout-limit.js Normal file
View File

@ -0,0 +1,786 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
const pointSleepSql = `
SELECT
pg_sleep(0.3),
'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator,
1 cartodb_id,
2 val
`;
const validationPointSleepSql = `
SELECT
pg_sleep(0.3),
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = pointSleepSql,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id',
attributes
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
attributes,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
const DATASOURCE_TIMEOUT_ERROR = {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
};
describe('user database timeout limit', function () {
describe('dataview', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but dataview request fails due to statement timeout', function (done) {
const params = {
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getDataview('count', params, (err, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
describe('raster', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching raster tiles', function () {
describe('with user\'s timeout of 200 ms', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.setUserDatabaseTimeoutLimit(200, (err) => {
if (err) {
return done(err);
}
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('"png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'png',
layers: [ 0 ]
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
it('"static png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png'
};
this.testClient.getStaticCenter(params, function (err, res, tile) {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.setUserDatabaseTimeoutLimit(200, (err) => {
if (err) {
return done(err);
}
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('"png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'png',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
it('"static png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getStaticCenter(params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
});
describe('vector', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching vector tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"mvt" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'mvt',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('interactivity', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching interactivity tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ interactivity: 'val' });
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"grid.json" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'grid.json',
layers: 'mapnik',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('torque', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
type: 'torque',
cartocss: TestClient.CARTOCSS.TORQUE
});
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'torque-layer0', index: 0, type: 'torque' }
}]
});
done();
});
});
});
describe('fetching torque tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
type: 'torque',
cartocss: TestClient.CARTOCSS.TORQUE
});
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"torque.json" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'torque.json',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
it('".png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'torque.png',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => {
assert.ifError(err);
assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('attributes:', function () {
describe('while validating in map instatiation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
attributes: {
id: 'cartodb_id',
columns: [ 'val' ]
}
});
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: {
id: 'layer0',
index: 0,
type: 'mapnik'
}
}]
});
done();
});
});
});
describe('fetching by feature id', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
attributes: {
id: 'cartodb_id',
columns: [ 'val' ]
}
});
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
featureId: 1,
layer: 0,
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getAttributes(params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
});

394
test/acceptance/user-render-timeout-limit.js Normal file
View File

@ -0,0 +1,394 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
const pointSleepSql = `
SELECT
pg_sleep(0.5),
'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator,
1 cartodb_id,
2 val
`;
// during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout
const validationPointSleepSql = `
SELECT
pg_sleep(0.5),
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = pointSleepSql,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id'
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
describe('user render timeout limit', function () {
describe('map instantiation => validation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details",
layer: {
id: "layer0",
index: 0,
type: "mapnik"
}
}]
});
done();
});
});
});
describe('raster', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but tile request fails due to render timeout', function (done) {
this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works and render tile fails', function (done) {
var params = {
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details"
}]
});
done();
});
});
});
});
describe('vector', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but vector tile request fails due to render timeout', function (done) {
const params = {
format: 'mvt',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
});
done();
});
});
});
describe('interativity', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) {
const params = {
layers: 'mapnik',
format: 'grid.json',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
});
done();
});
});
});
describe('static images', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but static image fails due to render timeout', function (done) {
const params = {
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png'
};
this.testClient.getStaticCenter(params, function (err, res, tile) {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works and render tile fails', function (done) {
const params = {
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getStaticCenter(params, function (err, res, timeoutError) {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details"
}]
});
done();
});
});
});
});
});

19
test/acceptance/widgets/ported/aggregation.js
View File

@ -322,6 +322,25 @@ describe('widgets', function() {
});
});
});
[adm0name].forEach(function(userQuery) {
it('should search with sum aggregation: ' + userQuery, function(done) {
this.testClient = new TestClient(aggregationSumMapConfig);
this.testClient.widgetSearch('adm0name', userQuery, function (err, res, searchResult) {
assert.ok(!err, err);
assert.ok(searchResult);
assert.equal(searchResult.type, 'aggregation');
assert.equal(searchResult.categories.length, 1);
assert.deepEqual(
searchResult.categories,
[{ category:"Argentina", value:28015640 }]
);
done();
});
});
});
});
});

108
test/acceptance/widgets/regressions.js
View File

@ -218,6 +218,114 @@ describe('widgets-regressions', function() {
});
});
it('should not count the polygons outside the bounding box', function(done) {
// $ % $ = not intersecting left triangle
// $$ **VVVVV** %% % = not intersecting right triangle
// $$$ *VVVVV* %%% * = intersecting triangle
// $$$$ ***** %%%% V = bounding box
// $$$$$ *** %%%%%
// $$$$$$ * %%%%%%
// $$$$$$$ %%%%%%%
// $$$$$$$$ %%%%%%%%
const notIntersectingLeftTriangle = {
type: "Polygon",
coordinates:[[
[-161.015625,69.28725695167886],
[-162.7734375,-7.710991655433217],
[-40.78125,-8.059229627200192],
[-161.015625,69.28725695167886]
]]
};
const notIntersectingRightTriangle = {
type: "Polygon",
coordinates: [[
[-29.179687499999996,-7.01366792756663],
[103.71093749999999,-6.664607562172573],
[105.46875,69.16255790810501],
[-29.179687499999996,-7.01366792756663]
]]
};
const intersectingTriangle = {
type: "Polygon",
coordinates:[[
[-117.42187500000001,68.13885164925573],
[-35.859375,20.96143961409684],
[59.4140625,68.52823492039876],
[-117.42187500000001,68.13885164925573]
]]
};
const query = `
SELECT
ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON(
'${JSON.stringify(notIntersectingLeftTriangle)}'
), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id, 'notIntersectingLeftTriangle' AS name
UNION
SELECT
ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON(
'${JSON.stringify(notIntersectingRightTriangle)}'
), 4326), 3857), 2, 'notIntersectingRightTriangle'
UNION
SELECT
ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON(
'${JSON.stringify(intersectingTriangle)}'
), 4326), 3857), 3, 'intersectingTriangle'
`;
const mapConfig = {
version: '1.5.0',
layers: [
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
dataviews: {
val_formula: {
source: {
id: 'a0'
},
type: 'aggregation',
options: {
column: "name",
aggregation: "count",
}
}
},
analyses: [
{
"id": "a0",
"type": "source",
"params": {
"query": query
}
}
]
};
this.testClient = new TestClient(mapConfig, 1234);
const params = {
bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658'
};
this.testClient.getDataview('val_formula', params, function(err, dataview) {
assert.ifError(err);
assert.equal(dataview.categories.length, 1);
assert.equal(dataview.categories[0].category, 'intersectingTriangle');
done();
});
});
});
});

307
test/acceptance/x_cache_channel.js
View File

@ -1,307 +0,0 @@
var testHelper = require('../support/test_helper');
var assert = require('../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var LayergroupToken = require('../support/layergroup-token');
describe('get requests x-cache-channel', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var statusOkResponse = {
status: 200
};
var mapConfig = {
version: '1.3.0',
layers: [
{
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
]
};
var layergroupRequest = {
url: '/api/v1/map?config=' + encodeURIComponent(JSON.stringify(mapConfig)),
method: 'GET',
headers: {
host: 'localhost'
}
};
function getRequest(url, addApiKey, callbackName) {
var params = {};
if (!!addApiKey) {
params.api_key = '1234';
}
if (!!callbackName) {
params.callback = callbackName;
}
return {
url: url + '?' + qs.stringify(params),
method: 'GET',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
};
}
function validateXCacheChannel(done, expectedCacheChannel) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(res.headers['x-cache-channel']);
if (expectedCacheChannel) {
assert.equal(res.headers['x-cache-channel'], expectedCacheChannel);
}
done();
};
}
function noXCacheChannelHeader(done) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(
!res.headers['x-cache-channel'],
'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
);
done();
};
}
function withLayergroupId(callback) {
assert.response(
server,
layergroupRequest,
statusOkResponse,
function(res, err) {
if (err) {
return callback(err);
}
var layergroupId = JSON.parse(res.body).layergroupid;
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
callback(null, layergroupId, res);
}
);
}
describe('header should be present', function() {
it('/api/v1/map Map instantiation', function(done) {
var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table');
withLayergroupId(function(err, layergroupId, res) {
testFn(res);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
});
describe('header should NOT be present', function() {
it('/', function(done) {
assert.response(
server,
getRequest('/'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/version', function(done) {
assert.response(
server,
getRequest('/version'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/health', function(done) {
assert.response(
server,
getRequest('/health'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/api/v1/map/named list named maps', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named', true),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
describe('with named maps', function() {
var templateName = 'x_cache';
beforeEach(function(done) {
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
layergroup: mapConfig
};
var namedMapRequest = {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(template)
};
assert.response(
server,
namedMapRequest,
statusOkResponse,
function(res, err) {
done(err);
}
);
});
afterEach(function(done) {
assert.response(
server,
{
url: '/api/v1/map/named/' + templateName + '?api_key=1234',
method: 'DELETE',
headers: {
host: 'localhost'
}
},
{
status: 204
},
function(res, err) {
done(err);
}
);
});
it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
});
});
});

BIN
test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.3 KiB

1
test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.geojson vendored Normal file
View File

@ -0,0 +1 @@
{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[-53839,4629161]},"properties":{"name":"Alicante","cartodb_id":1200}},{"type":"Feature","geometry":{"type":"Point","coordinates":[242835,5069332]},"properties":{"name":"Barcelona","cartodb_id":5330}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-5567,4861644]},"properties":{"name":"Castello","cartodb_id":1201}},{"type":"Feature","geometry":{"type":"Point","coordinates":[272735,5092314]},"properties":{"name":"Mataro","cartodb_id":615}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125787,4576600]},"properties":{"name":"Murcia","cartodb_id":952}},{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139148,5030112]},"properties":{"name":"Tarragona","cartodb_id":616}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-44746,4791667]},"properties":{"name":"Valencia","cartodb_id":5942}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-99072,5108695]},"properties":{"name":"Zaragoza","cartodb_id":5932}}]}

1
test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json vendored Normal file
View File

@ -0,0 +1 @@
{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !! ","!!! !!!!! ","!!!!!!! ! ","!!! !!!!! "," !! ! "," "," "," "," "," "," "," "," ### # "," ####### ###"," ####### ## ","$ ## #### ## ","$$ ","$$ ","$$ "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","9","2","1"],"data":{"1":{"cartodb_id":5942},"2":{"cartodb_id":5500},"9":{"cartodb_id":1201}}}

BIN
test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.4 KiB

1
test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json vendored Normal file
View File

@ -0,0 +1 @@
{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !!! ! "," !!!!!!! !!!"," !!!!!!! !! "," !! !!!! !! "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","1"],"data":{"1":{"cartodb_id":5500}}}

1
test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.geojson vendored Normal file
View File

@ -0,0 +1 @@
{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}}]}

BIN
test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt vendored Normal file
View File

Binary file not shown.

BIN
test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt vendored Normal file
View File

Binary file not shown.

BIN
test/fixtures/previews/populated_places_simple_reduced-override-bbox.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 94 KiB

31
test/support/assert.js
View File

@ -126,22 +126,25 @@ assert.response = function(server, req, res, callback) {
// Assert response body
if (res.body) {
var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
assert.ok(
eql,
colorize('[red]{Invalid response body.}\n' +
if (!eql) {
return callback(response, new Error(colorize(
'[red]{Invalid response body.}\n' +
' Expected: [green]{' + res.body + '}\n' +
' Got: [red]{' + response.body + '}')
);
' Got: [red]{' + response.body + '}'))
);
}
}
// Assert response status
if (typeof status === 'number') {
assert.equal(response.statusCode, status,
colorize('[red]{Invalid response status code.}\n' +
if (response.statusCode != status) {
return callback(response, new Error(colorize(
'[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body)
);
' Body: ' + response.body))
);
}
}
// Assert response headers
@ -152,11 +155,13 @@ assert.response = function(server, req, res, callback) {
actual = response.headers[name.toLowerCase()],
expected = res.headers[name],
headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
assert.ok(headerEql,
colorize('Invalid response header [bold]{' + name + '}.\n' +
if (!headerEql) {
return callback(response, new Error(colorize(
'Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expected + '}\n' +
' Got: [red]{' + actual + '}')
);
' Got: [red]{' + actual + '}'))
);
}
}
}

4
test/support/prepare_db.sh
View File

@ -75,8 +75,8 @@ if test x"$PREPARE_PGSQL" = xyes; then
dropdb "${TEST_DB}"
createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount'
CURL_ARGS=""
for i in ${REMOTE_SQL_SCRIPTS}

6
test/support/sql/cdb_invalidate_varnish.sql Normal file
View File

@ -0,0 +1,6 @@
CREATE OR REPLACE FUNCTION CDB_Invalidate_Varnish(table_name TEXT)
RETURNS void AS
$$
BEGIN
END;
$$ LANGUAGE PLPGSQL;

74
test/support/sql/windshaft.test.sql
View File

@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES
INSERT INTO _vovw_1_test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5);
-- table with overviews whit special float values
CREATE TABLE test_special_float_values_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id;
SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true);
ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass);
INSERT INTO test_special_float_values_table_overviews VALUES
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1),
(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1),
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1),
(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1);
ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id);
CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom);
CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator);
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
CREATE TABLE _vovw_1_test_special_float_values_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER;
INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2),
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2);
-- analysis tables -----------------------------------------------
@ -649,3 +721,5 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters int
END;
$$ LANGUAGE plpgsql;
GRANT ALL ON FUNCTION cdb_crankshaft.CDB_KMeans(text, integer, integer) TO :TESTUSER;
ANALYZE;

596
test/support/test-client.js
View File

@ -3,7 +3,8 @@
var qs = require('querystring');
var step = require('step');
var urlParser = require('url');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
var mapnik = require('windshaft').mapnik;
var LayergroupToken = require('./layergroup-token');
@ -14,16 +15,33 @@ var helper = require('./test_helper');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
serverOptions.analysis.batch.inlineExecution = true;
var server = new CartodbWindshaft(serverOptions);
function TestClient(mapConfig, apiKey) {
this.mapConfig = mapConfig;
const MAPNIK_SUPPORTED_FORMATS = {
'png': true,
'png32': true,
'grid.json': true,
'geojson': true,
'mvt': true
}
function TestClient(config, apiKey) {
this.mapConfig = isMapConfig(config) ? config : null;
this.template = isTemplate(config) ? config : null;
this.apiKey = apiKey;
this.keysToDelete = {};
this.server = new CartodbWindshaft(serverOptions);
}
module.exports = TestClient;
function isMapConfig(config) {
return config && config.layers;
}
function isTemplate(config) {
return config && config.layergroup;
}
module.exports.RESPONSE = {
ERROR: {
status: 400,
@ -63,9 +81,42 @@ module.exports.CARTOCSS = {
' line-width: 0.5;',
' line-opacity: 1;',
'}'
].join('\n'),
TORQUE: [
'Map {',
' -torque-frame-count: 256;',
' -torque-animation-duration: 30;',
' -torque-time-attribute: "cartodb_id";',
' -torque-aggregation-function: "count(1)";',
' -torque-resolution: 4;',
' -torque-data-aggregation: linear;',
'}',
'#layer {',
' marker-width: 7;',
' marker-fill: #FFB927;',
' marker-fill-opacity: 0.9;',
' marker-line-width: 1;',
' marker-line-color: #FFF;',
' marker-line-opacity: 1;',
' comp-op: lighter;',
'}',
'#layer[frame-offset=1] {',
' marker-width: 9;',
' marker-fill-opacity: 0.45;',
'}',
'#layer[frame-offset=2] {',
' marker-width: 11;',
' marker-fill-opacity: 0.225;',
'}'
].join('\n')
};
module.exports.SQL = {
EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator',
ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator'
}
TestClient.prototype.getWidget = function(widgetName, params, callback) {
var self = this;
@ -83,7 +134,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -142,7 +193,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -194,7 +245,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -251,7 +302,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
}
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -318,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -360,7 +411,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
};
['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) {
['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) {
if (params.hasOwnProperty(extraParam)) {
urlParams[extraParam] = params[extraParam];
}
@ -371,7 +422,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
}
url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -390,9 +441,115 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
);
},
function finish(err, dataview) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
return callback(err, dataview);
if (err) {
return callback(err);
}
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(null, dataview);
}
);
};
TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, callback) {
var self = this;
if (!callback) {
callback = params;
params = {};
}
var extraParams = {};
if (this.apiKey) {
extraParams.api_key = this.apiKey;
}
if (params && params.filters) {
extraParams.filters = JSON.stringify(params.filters);
}
var url = '/api/v1/map';
if (Object.keys(extraParams).length > 0) {
url += '?' + qs.stringify(extraParams);
}
var expectedResponse = params.response || {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
var layergroupId;
step(
function createLayergroup() {
var next = this;
assert.response(self.server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return next(null, parsedBody.layergroupid);
}
);
},
function getFeatureAttributes(err, layergroupId) {
assert.ifError(err);
var next = this;
url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId;
assert.response(self.server,
{
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
},
expectedResponse,
function(res, err) {
if (err) {
return next(err);
}
next(null, JSON.parse(res.body));
}
);
},
function finish(err, attributes) {
if (err) {
return callback(err);
}
return callback(null, attributes);
}
);
};
@ -406,24 +563,77 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
var url = '/api/v1/map';
var urlNamed = url + '/named';
if (this.apiKey) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
var layergroupId;
if (params.layergroupid) {
layergroupId = params.layergroupid
}
step(
function createLayergroup() {
function createTemplate () {
var next = this;
assert.response(server,
if (!self.template) {
return next();
}
if (!self.apiKey) {
return next(new Error('apiKey param is mandatory to create a new template'));
}
params.placeholders = params.placeholders || {};
assert.response(self.server,
{
url: url,
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
data: JSON.stringify(self.template)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function (res, err) {
if (err) {
return next(err);
}
return next(null, JSON.parse(res.body).template_id);
}
);
},
function createLayergroup(err, templateId) {
var next = this;
if (layergroupId) {
return next(null, layergroupId);
}
var data = templateId ? params.placeholders : self.mapConfig
var path = templateId ?
urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) :
url;
assert.response(self.server,
{
url: path,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(data)
},
{
status: 200,
@ -456,6 +666,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
var format = params.format || 'png';
if (layers === undefined && !MAPNIK_SUPPORTED_FORMATS[format]) {
throw new Error(`Missing layer filter while fetching ${format} tile, review params argument`);
}
url += [z,x,y].join('/');
url += '.' + format;
@ -471,37 +685,76 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
};
var expectedResponse = {
var expectedResponse = Object.assign({}, {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
'Content-Type': 'image/png'
}
};
}, params.response);
var isPng = format.match(/png$/);
if (isPng) {
request.encoding = 'binary';
expectedResponse.headers['Content-Type'] = 'image/png';
}
assert.response(server, request, expectedResponse, function(res, err) {
var isMvt = format.match(/mvt$/);
if (isMvt) {
request.encoding = 'binary';
if (expectedResponse.status === 200) {
expectedResponse.headers['Content-Type'] = 'application/x-protobuf';
}
}
var isGeojson = format.match(/geojson$/);
if (isGeojson) {
request.encoding = 'utf-8';
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
var isGridJSON = format.match(/grid.json$/);
if (isGridJSON) {
request.encoding = 'utf-8';
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
if (params.contentType) {
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var obj;
if (isPng) {
obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
} else {
obj = JSON.parse(res.body);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/x-protobuf':
body = new mapnik.VectorTile(z, x, y);
body.setDataSync(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, obj);
next(null, res, body);
});
},
function finish(err, res, image) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, image);
}
);
@ -526,7 +779,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -538,18 +791,124 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
},
expectedResponse,
function(res, err) {
// If there is a response, we are still interested in catching the created keys
// to be able to delete them on the .drain() method.
if (res) {
var parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
}
if (err) {
return callback(err);
}
var parsedBody = JSON.parse(res.body);
return callback(null, parsedBody);
}
);
};
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
TestClient.prototype.getStaticCenter = function (params, callback) {
var self = this;
let { layergroupid, z, lat, lng, width, height, format } = params
var url = `/api/v1/map/`;
if (this.apiKey) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
step(
function createLayergroup() {
var next = this;
if (layergroupid) {
return next(null, layergroupid);
}
return callback(null, parsedBody);
var data = self.mapConfig
var path = url;
assert.response(self.server,
{
url: path,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(data)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
return next(null, JSON.parse(res.body).layergroupid);
}
);
},
function getStaticResult(err, _layergroupid) {
assert.ifError(err);
var next = this;
layergroupid = _layergroupid;
url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}`
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
}
var request = {
url: url,
encoding: 'binary',
method: 'GET',
headers: {
host: 'localhost'
}
};
var expectedResponse = Object.assign({}, {
status: 200,
headers: {
'Content-Type': 'image/png'
}
}, params.response);
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, body);
});
},
function finish(err, res, image) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, image);
}
);
};
@ -568,7 +927,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -629,7 +988,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
}
};
assert.response(server, request, expectedResponse, function(res, err) {
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
next(null, res, JSON.parse(res.body));
});
@ -642,11 +1001,119 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
);
};
TestClient.prototype.getAttributes = function(params, callback) {
var self = this;
if (!Number.isFinite(params.featureId)) {
throw new Error('featureId param must be a number')
}
if (!Number.isFinite(params.layer)) {
throw new Error('layer param must be a number')
}
var url = '/api/v1/map';
if (this.apiKey) {
url += '?' + qs.stringify({ api_key: this.apiKey });
}
var layergroupid;
if (params.layergroupid) {
layergroupid = params.layergroupid
}
step(
function createLayergroup() {
var next = this;
if (layergroupid) {
return next(null, layergroupid);
}
assert.response(self.server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
return next(null, parsedBody.layergroupid);
}
);
},
function getAttributes(err, _layergroupid) {
assert.ifError(err);
var next = this;
layergroupid = _layergroupid;
url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`;
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
}
var request = {
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
};
var expectedResponse = params.response || {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
assert.response(self.server, request, expectedResponse, function (res, err) {
assert.ifError(err);
var attributes = JSON.parse(res.body);
next(null, res, attributes);
});
},
function finish(err, res, attributes) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, attributes);
}
);
};
TestClient.prototype.drain = function(callback) {
helper.deleteRedisKeys(this.keysToDelete, callback);
};
module.exports.getStaticMap = function getStaticMap(templateName, params, callback) {
var self = this;
self.server = new CartodbWindshaft(serverOptions);
if (!callback) {
callback = params;
params = null;
@ -677,9 +1144,56 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
assert.response(self.server, requestOptions, expectedResponse, function (res, err) {
helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() {
return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary')));
});
});
};
TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) {
const userTimeoutLimitsKey = `limits:timeout:${user}`;
const params = [
userTimeoutLimitsKey,
'render', userTimeoutLimit,
'render_public', userTimeoutLimit
];
this.keysToDelete[userTimeoutLimitsKey] = 5;
helper.configureMetadata('hmset', params, callback);
};
TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) {
const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db';
const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 })
const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 })
const publicuser = global.environment.postgres.user;
// we need to guarantee all new connections have the new settings
helper.cleanPGPoolConnections()
const psql = new PSQL({
user: 'postgres',
dbname: dbname,
host: global.environment.postgres.host,
port: global.environment.postgres.port
});
step(
function configureTimeouts () {
const timeoutSQLs = [
`ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`
];
const group = this.group();
timeoutSQLs.forEach(sql => psql.query(sql, group()));
},
callback
);
};

26
test/support/test_helper.js
View File

@ -14,6 +14,7 @@ var lzmaWorker = new LZMA();
var redis = require('redis');
var nock = require('nock');
var log4js = require('log4js');
var pg = require('pg');
// set environment specific variables
global.environment = require(__dirname + '/../../config/environments/test');
@ -127,6 +128,11 @@ afterEach(function(done) {
});
});
function cleanPGPoolConnections () {
// TODO: this method will be replaced by psql.end
pg.end();
}
function deleteRedisKeys(keysToDelete, callback) {
if (Object.keys(keysToDelete).length === 0) {
@ -166,12 +172,30 @@ function rmdirRecursiveSync(dirname) {
}
}
function configureMetadata(action, params, callback) {
redisClient.SELECT(5, function (err) {
if (err) {
return callback(err);
}
redisClient[action](params, function (err) {
if (err) {
return callback(err);
}
return callback();
});
});
}
module.exports = {
deleteRedisKeys: deleteRedisKeys,
lzma_compress_to_base64: lzma_compress_to_base64,
checkNoCache: checkNoCache,
checkSurrogateKey: checkSurrogateKey,
checkCache: checkCache,
rmdirRecursiveSync: rmdirRecursiveSync
rmdirRecursiveSync: rmdirRecursiveSync,
configureMetadata,
cleanPGPoolConnections
};

153
test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js Normal file
View File

@ -0,0 +1,153 @@
var assert = require('assert');
var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
function getDbConnectionMock () {
return {
query: function(sql, callback) {
return callback(null, {
rows: [{rows: 1}]
});
}
};
}
describe('mapnik-layer-stats', function() {
beforeEach(function () {
this.dbConnectionMock = getDbConnectionMock();
this.rendererCacheMock = {};
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
}
]
};
var testMapConfigTwoLayers = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
]
};
var testMapConfigOneLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
]
};
var testMapConfigTwoLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_3', 'test_table_4']
}
},
]
};
it('should return 1 feature for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayers);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
it('should return 1 feature for one layers with two tables', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers and two tables', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
});

36
test/unit/cartodb/backends/layer-stats/torque-layer-stats.js Normal file
View File

@ -0,0 +1,36 @@
var assert = require('assert');
var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
describe('torque-layer-stats', function () {
beforeEach(function () {
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'torque',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
}
},
]
};
it('should return torque stats for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layerId = 0;
var layer = mapConfig.getLayer(layerId);
var testSubject = new TorqueLayerStats();
testSubject.getStats(layer, {}, function (err, result) {
assert.ifError(err);
assert.deepEqual({}, result);
done();
});
});
});

44
test/unit/cartodb/backends/turbo-carto-postgres-datasource.js Normal file
View File

@ -0,0 +1,44 @@
var PostgresDatasource = require('../../../../lib/cartodb/backends/turbo-carto-postgres-datasource');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
var assert = require('assert');
describe('turbo-carto-postgres-datasource', function() {
beforeEach(function () {
const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db';
const psql = new PSQL({
user: 'postgres',
dbname: dbname,
host: global.environment.postgres.host,
port: global.environment.postgres.port
});
const sql = [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS values',
'FROM generate_series(1, 1000) x'
].join('\n');
this.datasource = new PostgresDatasource(psql, sql);
});
it('should ignore NaNs and Infinities when computing ramps', function(done) {
var column = 'values';
var buckets = 4;
var method = 'equal';
this.datasource.getRamp(column, buckets, method, function(err, result) {
var expected_result = {
ramp: [ 252, 501, 750, 999 ],
stats: { min_val: 3, max_val: 999, avg_val: 501 },
strategy: undefined
};
assert.deepEqual(result, expected_result);
done();
});
});
});

36
test/unit/cartodb/lzmaMiddleware.test.js Normal file
View File

@ -0,0 +1,36 @@
var assert = require('assert');
var testHelper = require('../../support/test_helper');
var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma');
describe('lzma-middleware', function() {
it('it should extend params with decoded lzma', function(done) {
var qo = {
config: {
version: '1.3.0'
}
};
testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
var req = {
headers: {
host:'localhost'
},
query: {
api_key: 'test',
lzma: data
}
};
lzmaMiddleware(req, {}, function(err) {
if ( err ) {
return done(err);
}
var query = req.query;
assert.deepEqual(qo.config, query.config);
assert.equal('test', query.api_key);
done();
});
});
});
});

17
test/unit/cartodb/model/filter/bbox-filters.test.js
View File

@ -111,6 +111,23 @@ describe('Bounding box filter', function() {
createRef([-180, -45, 0, 45])
);
});
it('generating multiple bbox', function() {
var bbox = [90, -45, 190, 45];
var bboxFilter = createFilter(bbox);
assert.equal(bboxFilter.bboxes.length, 2);
assert.deepEqual(
bboxFilter.bboxes[0],
createRef([90, -45, 180, 45])
);
assert.deepEqual(
bboxFilter.bboxes[1],
createRef([-180, -45, -170, 45])
);
});
});
describe('out of bounds', function() {

11
test/unit/cartodb/ported/tile_stats.test.js
View File

@ -6,10 +6,13 @@ var LayergroupController = require('../../../../lib/cartodb/controllers/layergro
describe('tile stats', function() {
after(function() {
global.statsClient = null;
beforeEach(function () {
this.statsClient = global.statsClient;
});
afterEach(function() {
global.statsClient = this.statsClient;
});
it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() {
var expectedCalls = 2, // it will call increment once for the general error
@ -26,12 +29,14 @@ describe('tile stats', function() {
var layergroupController = new LayergroupController();
var reqMock = {
profiler: { toJSONString:function() {} },
params: {
format: invalidFormat
}
};
var resMock = {
status: function() { return this; },
set: function() {},
json: function() {},
jsonp: function() {},
send: function() {}
@ -54,12 +59,14 @@ describe('tile stats', function() {
}
});
var reqMock = {
profiler: { toJSONString:function() {} },
params: {
format: validFormat
}
};
var resMock = {
status: function() { return this; },
set: function() {},
json: function() {},
jsonp: function() {},
send: function() {}

51
test/unit/cartodb/req2params.test.js
View File

@ -1,6 +1,6 @@
var assert = require('assert');
var _ = require('underscore');
var test_helper = require('../../support/test_helper');
require('../../support/test_helper');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
@ -98,34 +98,31 @@ describe('req2params', function() {
});
});
it('it should extend params with decoded lzma', function(done) {
var qo = {
config: {
version: '1.3.0'
it('it should remove invalid params', function(done) {
var config = {
version: '1.3.0'
};
var req = {
headers: {
host:'localhost'
},
query: {
non_included: 'toberemoved',
api_key: 'test',
style: 'override',
config: config
}
};
test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
var req = {
headers: {
host:'localhost'
},
query: {
non_included: 'toberemoved',
api_key: 'test',
style: 'override',
lzma: data
}
};
baseController.req2params(prepareRequest(req), function(err, req) {
if ( err ) {
return done(err);
}
var query = req.params;
assert.deepEqual(qo.config, query.config);
assert.equal('test', query.api_key);
assert.equal(undefined, query.non_included);
done();
});
baseController.req2params(prepareRequest(req), function(err, req) {
if (err) {
return done(err);
}
var query = req.params;
assert.deepEqual(config, query.config);
assert.equal('test', query.api_key);
assert.equal(undefined, query.non_included);
assert.equal(undefined, query.style);
done();
});
});

905
yarn.lock
View File

File diff suppressed because it is too large Load Diff