cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into new_querytables

Browse Source
This commit is contained in:
Alejandro Martínez 2016-02-15 11:47:29 +00:00
commit 9e7b288f44
29 changed files with 1409 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
NEWS.md
View File

@ -1,10 +1,18 @@
# Changelog
## 2.22.1
## 2.23.1
Released 2016-mm-dd
## 2.23.0
Released 2016-02-10
Improvements:
- Support for overviews
## 2.22.0
Released 2016-02-08

44
lib/cartodb/api/overviews_metadata_api.js Normal file
View File

@ -0,0 +1,44 @@
function OverviewsMetadataApi(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
module.exports = OverviewsMetadataApi;
// TODO: share this with QueryTablesApi? ... or maintain independence?
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
function prepareSql(sql) {
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1')
;
}
OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) {
var query = 'SELECT * FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not get overviews metadata: ' + msg));
return;
}
var metadata = {};
rows.forEach(function(row) {
var table = row.base_table;
var table_metadata = metadata[table];
if ( !table_metadata ) {
table_metadata = metadata[table] = {};
}
table_metadata[row.z] = { table: row.overview_table };
});
return callback(null, metadata);
});
};

3
lib/cartodb/controllers/base.js
View File

@ -239,6 +239,7 @@ module.exports.findStatusCode = findStatusCode;
function statusFromErrorMessage(errMsg) {
// Find an appropriate statusCode based on message
// jshint maxcomplexity:7
var statusCode = 400;
if ( -1 !== errMsg.indexOf('permission denied') ) {
statusCode = 403;
@ -252,6 +253,8 @@ function statusFromErrorMessage(errMsg) {
else if ( -1 !== errMsg.indexOf('does not exist') ) {
if ( -1 !== errMsg.indexOf(' role ') ) {
statusCode = 403; // role 'xxx' does not exist
} else if ( errMsg.match(/function .* does not exist/) ) {
statusCode = 400; // invalid SQL (SQL function does not exist)
} else {
statusCode = 404;
}

41
lib/cartodb/controllers/map.js
View File

@ -18,6 +18,7 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter');
/**
* @param {AuthApi} authApi
@ -26,12 +27,14 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye
* @param {MapBackend} mapBackend
* @param metadataBackend
* @param {QueryTablesApi} queryTablesApi
* @param {OverviewsMetadataApi} overviewsMetadataApi
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi,
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
queryTablesApi, overviewsMetadataApi,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
@ -41,11 +44,13 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.mapBackend = mapBackend;
this.metadataBackend = metadataBackend;
this.queryTablesApi = queryTablesApi;
this.overviewsMetadataApi = overviewsMetadataApi;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTables = layergroupAffectedTables;
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi);
}
util.inherits(MapController, BaseController);
@ -148,6 +153,22 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
}
);
},
function addOverviewsInformation(err, requestMapConfig, datasource) {
assert.ifError(err);
var next = this;
self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
function(err, layers) {
if (err) {
return next(err);
}
if (layers) {
requestMapConfig.layers = layers;
}
return next(null, requestMapConfig, datasource);
}
);
},
function createLayergroup(err, requestMapConfig, datasource) {
assert.ifError(err);
mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource());
@ -203,7 +224,23 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
);
mapConfigProvider.getMapConfig(this);
},
function createLayergroup(err, mapConfig_, rendererParams/*, context*/) {
function addOverviewsInformation(err, requestMapConfig, rendererParams/*, context*/) {
assert.ifError(err);
var next = this;
self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
function(err, layers) {
if (err) {
return next(err);
}
if (layers) {
requestMapConfig.layers = layers;
}
return next(null, requestMapConfig, rendererParams);
}
);
},
function createLayergroup(err, mapConfig_, rendererParams) {
assert.ifError(err);
mapConfig = mapConfig_;
self.mapBackend.createLayergroup(

53
lib/cartodb/models/mapconfig_overviews_adapter.js Normal file
View File

@ -0,0 +1,53 @@
var queue = require('queue-async');
var _ = require('underscore');
function MapConfigOverviewsAdapter(overviewsMetadataApi) {
this.overviewsMetadataApi = overviewsMetadataApi;
}
module.exports = MapConfigOverviewsAdapter;
MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) {
var self = this;
if (!layers || layers.length === 0) {
return callback(null, layers);
}
var augmentLayersQueue = queue(layers.length);
function augmentLayer(layer, done) {
if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) {
return done(null, layer);
}
self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){
if (err) {
done(err, layer);
} else {
if ( !_.isEmpty(metadata) ) {
layer = _.extend({}, layer);
layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } });
}
done(null, layer);
}
});
}
function layersAugmentQueueFinish(err, layers) {
if (err) {
return callback(err);
}
if (!layers || layers.length === 0) {
return callback(new Error('Missing layers array from layergroup config'));
}
return callback(null, layers);
}
layers.forEach(function(layer) {
augmentLayersQueue.defer(augmentLayer, layer);
});
augmentLayersQueue.awaitAll(layersAugmentQueueFinish);
};

3
lib/cartodb/server.js
View File

@ -20,6 +20,7 @@ var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api');
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
@ -52,6 +53,7 @@ module.exports = function(serverOptions) {
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var queryTablesApi = new QueryTablesApi(pgQueryRunner);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
@ -175,6 +177,7 @@ module.exports = function(serverOptions) {
mapBackend,
metadataBackend,
queryTablesApi,
overviewsMetadataApi,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache

7
lib/cartodb/server_options.js
View File

@ -1,5 +1,10 @@
var os = require('os');
var _ = require('underscore');
var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
});
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
@ -15,6 +20,8 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, {
http: {}
});
rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter;
// Perform keyword substitution in statsd
// See https://github.com/CartoDB/Windshaft-cartodb/issues/153
if ( global.environment.statsd ) {

182
lib/cartodb/utils/overviews_query_rewriter.js Normal file
View File

@ -0,0 +1,182 @@
var TableNameParser = require('./table_name_parser');
function OverviewsQueryRewriter(options) {
this.options = options;
}
module.exports = OverviewsQueryRewriter;
// TODO: some names are introudced in the queries, and the
// '_vovw_' (for vector overviews) is used in them, but no check
// is performed for conflicts with existing identifiers in the query.
// Build UNION expression to replace table, using overviews metadata
// overviews metadata: { 1: 'table_ov1', ... }
// assume table and overview names include schema if necessary and are quoted as needed
function overviews_view_for_table(table, overviews_metadata, indent) {
var condition, i, len, ov_table, overview_layers, selects, z_hi, z_lo;
var parsed_table = TableNameParser.parse(table);
var sorted_overviews = []; // [[1, 'table_ov1'], ...]
indent = indent || ' ';
for (var z in overviews_metadata) {
if (overviews_metadata.hasOwnProperty(z)) {
sorted_overviews.push([z, overviews_metadata[z].table]);
}
}
sorted_overviews.sort(function(a, b){ return a[0]-b[0]; });
overview_layers = [];
z_lo = null;
for (i = 0, len = sorted_overviews.length; i < len; i++) {
z_hi = parseInt(sorted_overviews[i][0]);
ov_table = sorted_overviews[i][1];
overview_layers.push([overview_z_condition(z_lo, z_hi), ov_table]);
z_lo = z_hi;
}
overview_layers.push(["_vovw_z > " + z_lo, table]);
selects = overview_layers.map(function(condition_table) {
condition = condition_table[0];
ov_table = TableNameParser.parse(condition_table[1]);
ov_table.schema = ov_table.schema || parsed_table.schema;
var ov_identifier = TableNameParser.table_identifier(ov_table);
return indent + "SELECT * FROM " + ov_identifier + ", _vovw_scale WHERE " + condition;
});
return selects.join("\n"+indent+"UNION ALL\n");
}
function overview_z_condition(z_lo, z_hi) {
if (z_lo !== null) {
if (z_lo === z_hi - 1) {
return "_vovw_z = " + z_hi;
} else {
return "_vovw_z > " + z_lo + " AND _vovw_z <= " + z_hi;
}
} else {
if (z_hi === 0) {
return "_vovw_z = " + z_hi;
} else {
return "_vovw_z <= " + z_hi;
}
}
}
// name to be used for the view of the table using overviews
function overviews_view_name(table) {
var parsed_table = TableNameParser.parse(table);
parsed_table.table = '_vovw_' + parsed_table.table;
parsed_table.schema = null;
return TableNameParser.table_identifier(parsed_table);
}
// replace a table name in a query by anoter name
function replace_table_in_query(sql, old_table_name, new_table_name) {
var old_table = TableNameParser.parse(old_table_name);
var new_table = TableNameParser.parse(new_table_name);
var old_table_ident = TableNameParser.table_identifier(old_table);
var new_table_ident = TableNameParser.table_identifier(new_table);
// text that will be substituted by the table name pattern
var replacement = new_table_ident;
// regular expression prefix (beginning) to match a table name
function pattern_prefix(schema, identifier) {
if ( schema ) {
// to match a table name including schema prefix
// name should not be part of another name, so we require
// to start a at a word boundary
if ( identifier[0] !== '"' ) {
return '\\b';
} else {
return '';
}
} else {
// to match a table name without schema
// name should not begin right after a dot (i.e. have a explicit schema)
// nor be part of another name
// since the pattern matches the first character of the table
// it must be put back in the replacement text
replacement = '$01'+replacement;
return '([^\.a-z0-9_]|^)';
}
}
// regular expression suffix (ending) to match a table name
function pattern_suffix(identifier) {
// name shouldn't be the prefix of a longer name
if ( identifier[identifier.length-1] !== '"' ) {
return '\\b';
} else {
return '';
}
}
// regular expression to match a table name
var regexp = pattern_prefix(old_table.schema, old_table_ident) +
old_table_ident +
pattern_suffix(old_table_ident);
// replace all occurrences of the table pattern
return sql.replace(new RegExp(regexp, 'g'), replacement);
}
function overviews_query(query, overviews, zoom_level_expression) {
var replaced_query = query;
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
for ( var table in overviews ) {
if (overviews.hasOwnProperty(table)) {
var table_overviews = overviews[table];
var table_view = overviews_view_name(table);
replaced_query = replace_table_in_query(replaced_query, table, table_view);
sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )";
}
}
if ( replaced_query !== query ) {
sql += "\n";
sql += replaced_query;
} else {
sql = query;
}
return sql;
}
// Transform an SQL query so that it uses overviews.
// overviews contains metadata about the overviews to be used:
// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
//
// For a given query `SELECT * FROM table`, if any of tables in it
// has overviews as defined by the provided metadat, the query will
// be transform into something similar to this:
//
// WITH _vovw_scale AS ( ... ), -- define scale level
// WITH _vovw_table AS ( ... ), -- define union of overviews and base table
// SELECT * FROM _vovw_table -- query with table replaced by _vovw_table
//
// This transformation can in principle be applied to arbitrary queries
// (except for the case of queries that include the name of tables with
// overviews inside text literals: at the current table name substitution
// doesnn't prevent substitution inside literals).
// But the transformation will currently only be applied to simple queries
// of the form detected by the overviews_supported_query function.
OverviewsQueryRewriter.prototype.query = function(query, data) {
var overviews = this.overviews_metadata(data);
if ( !overviews || !this.is_supported_query(query)) {
return query;
}
var zoom_level_expression = this.options.zoom_level || '0';
return overviews_query(query, overviews, zoom_level_expression);
};
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
return !!sql.match(
/^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
);
};
OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {
return data && data.overviews;
};

106
lib/cartodb/utils/table_name_parser.js Normal file
View File

@ -0,0 +1,106 @@
// Quote an PostgreSQL identifier if ncecessary
function quote_identifier_if_needed(txt) {
if ( txt && !txt.match(/^[a-z_][a-z_0-9]*$/)) {
return '"' + txt.replace(/\"/g, '""') + '"';
} else {
return txt;
}
}
// Parse PostgreSQL table name (possibly quoted and with optional schema).+
// Returns { schema: 'schema_name', table: 'table_name' }
function parse_table_name(table) {
function split_as_quoted_parts(table_name) {
// parse table into 'parts' that may be quoted, each part
// in the parts array being an object { part: 'text', quoted: false/true }
var parts = [];
var splitted = table_name.split(/\"/);
for (var i=0; i<splitted.length; i++ ) {
if ( splitted[i] === '' ) {
if ( parts.length > 0 && i < splitted.length-1 ) {
i++;
parts[parts.length - 1].part += '"' + splitted[i];
}
}
else {
var is_quoted = (i > 0 && splitted[i-1] === '') ||
(i < splitted.length - 1 && splitted[i+1] === '');
parts.push({ part: splitted[i], quoted: is_quoted });
}
}
return parts;
}
var parts = split_as_quoted_parts(table);
function split_single_part(part) {
var schema_part = null;
var table_part = null;
if ( part.quoted ) {
table_part = part.part;
} else {
var parts = part.part.split('.');
if ( parts.length === 1 ) {
schema_part = null;
table_part = parts[0];
} else if ( parts.length === 2 ) {
schema_part = parts[0];
table_part = parts[1];
} // else invalid table name
}
return {
schema: schema_part,
table: table_part
};
}
function split_two_parts(part1, part2) {
var schema_part = null;
var table_part = null;
if ( part1.quoted && !part2.quoted ) {
if ( part2.part[0] === '.' ) {
schema_part = part1.part;
table_part = part2.part.slice(1);
} // else invalid table name (missing dot)
} else if ( !part1.quoted && part2.quoted ) {
if ( part1.part[part1.part.length - 1] === '.' ) {
schema_part = part1.part.slice(0, -1);
table_part = part2.part;
} // else invalid table name (missing dot)
} // else invalid table name (missing dot)
return {
schema: schema_part,
table: table_part
};
}
if ( parts.length === 1 ) {
return split_single_part(parts[0]);
} else if ( parts.length === 2 ) {
return split_two_parts(parts[0], parts[1]);
} else if ( parts.length === 3 && parts[1].part === '.' ) {
return {
schema: parts[0].part,
table: parts[2].part
};
} // else invalid table name
}
function table_identifier(parsed_name) {
if ( parsed_name && parsed_name.table ) {
if ( parsed_name.schema ) {
return quote_identifier_if_needed(parsed_name.schema) + '.' + quote_identifier_if_needed(parsed_name.table);
} else {
return quote_identifier_if_needed(parsed_name.table);
}
} else {
return null;
}
}
module.exports = {
parse: parse_table_name,
quote: quote_identifier_if_needed,
table_identifier: table_identifier
};

15
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "2.22.0",
"version": "2.23.1",
"dependencies": {
"body-parser": {
"version": "1.14.2",
@ -85,7 +85,7 @@
},
"mime-types": {
"version": "2.1.9",
"from": "mime-types@>=2.1.2 <2.2.0",
"from": "mime-types@>=2.1.9 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
@ -193,7 +193,7 @@
"dependencies": {
"mime-types": {
"version": "2.1.9",
"from": "mime-types@>=2.1.9 <2.2.0",
"from": "mime-types@>=2.1.6 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
@ -382,7 +382,7 @@
},
"mime-types": {
"version": "2.1.9",
"from": "mime-types@>=2.1.9 <2.2.0",
"from": "mime-types@>=2.1.6 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
@ -536,7 +536,7 @@
},
"inherits": {
"version": "2.0.1",
"from": "inherits@>=2.0.0 <3.0.0",
"from": "inherits@>=2.0.1 <2.1.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
},
"isarray": {
@ -830,9 +830,8 @@
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
},
"windshaft": {
"version": "1.8.3",
"from": "windshaft@1.8.3",
"resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.8.3.tgz",
"version": "1.9.0",
"from": "windshaft@1.9.0",
"dependencies": {
"mapnik": {
"version": "1.4.15-cdb6",

4
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "2.22.1",
"version": "2.23.1",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@ -26,7 +26,7 @@
"node-statsd": "~0.0.7",
"underscore" : "~1.6.0",
"dot": "~1.0.2",
"windshaft": "1.8.3",
"windshaft": "1.9.0",
"step": "~0.0.6",
"queue-async": "~1.0.7",
"request": "~2.62.0",

37
test/acceptance/multilayer.js
View File

@ -576,7 +576,7 @@ describe(suiteName, function() {
version: '1.0.0',
layers: [
{ options: {
sql: 'select 1 as cartodb_id, !pixel_height! as h' +
sql: 'select 1 as cartodb_id, !pixel_height! as h,' +
'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
cartocss: '#layer { polygon-fit:red; }',
cartocss_version: '2.0.1'
@ -617,7 +617,7 @@ describe(suiteName, function() {
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 404, res.statusCode + ": " + res.body);
assert.equal(res.statusCode, 400, res.statusCode + ": " + res.body);
var parsed = JSON.parse(res.body);
var msg = parsed.errors[0];
assert.ok(msg.match(/bogus.*exist/), msg);
@ -1319,6 +1319,39 @@ describe(suiteName, function() {
);
});
it('should response to empty layers mapconfig', function(done) {
var layergroup = {
layers: []
};
assert.response(
server,
{
url: '/api/v1/map',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 200
},
function(res, err) {
assert.ok(!err);
var parsedBody = JSON.parse(res.body);
assert.ok(parsedBody.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done();
}
);
});
});
});

6
test/acceptance/multilayer_server.js
View File

@ -332,13 +332,9 @@ describe('tests from old api translated to multilayer', function() {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
// TODO when affected tables query makes the request to fail layergroup should be removed
keysToDelete['map_cfg|4fb7bd7008322ce66f22d20aebba1ab0'] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed, {
errors: ["Error: could not fetch affected tables or last updated time: fake error message"]
errors: ["could not get overviews metadata: fake error message"]
});
done();

110
test/acceptance/overviews_metadata.js Normal file
View File

@ -0,0 +1,110 @@
var test_helper = require('../support/test_helper');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
var RedisPool = require('redis-mpool');
var step = require('step');
var windshaft = require('windshaft');
describe('overviews metadata', function() {
// configure redis pool instance to use in tests
var redisPool = new RedisPool(global.environment.redis);
var overviews_layer = {
type: 'cartodb',
options: {
sql: 'SELECT * FROM test_table_overviews',
cartocss: '#layer { marker-fill: black; }',
cartocss_version: '2.3.0'
}
};
var non_overviews_layer = {
type: 'cartodb',
options: {
sql: 'SELECT * FROM test_table',
cartocss: '#layer { marker-fill: black; }',
cartocss_version: '2.3.0'
}
};
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
test_helper.deleteRedisKeys(keysToDelete, done);
});
it("layers with and without overviews", function(done) {
var layergroup = {
version: '1.0.0',
layers: [overviews_layer, non_overviews_layer]
};
var layergroup_url = '/api/v1/map';
var expected_token;
step(
function do_post()
{
var next = this;
assert.response(server, {
url: layergroup_url,
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
expected_token = parsedBody.layergroupid;
next(null, res);
});
},
function do_get_mapconfig(err)
{
assert.ifError(err);
var next = this;
var mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: 500000
});
mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) {
assert.ifError(err);
assert.deepEqual(non_overviews_layer, mapConfig._cfg.layers[1]);
assert.equal(mapConfig._cfg.layers[0].type, 'cartodb');
assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data);
var expected_data = {
overviews: {
test_table_overviews: {
1: { table: '_vovw_1_test_table_overviews' },
2: { table: '_vovw_2_test_table_overviews' }
}
}
};
assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data);
});
next(err);
},
function finish(err) {
keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done(err);
}
);
});
});

72
test/acceptance/overviews_queries.js Normal file
View File

@ -0,0 +1,72 @@
var testHelper = require('../support/test_helper');
var assert = require('../support/assert');
var cartodbServer = require('../../lib/cartodb/server');
var ServerOptions = require('./ported/support/ported_server_options');
var testClient = require('./ported/support/test_client');
var BaseController = require('../../lib/cartodb/controllers/base');
describe('overviews_queries', function() {
var server = cartodbServer(ServerOptions);
server.setMaxListeners(0);
var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2;
var req2paramsFn;
before(function() {
req2paramsFn = BaseController.prototype.req2params;
BaseController.prototype.req2params = ServerOptions.req2params;
});
after(function() {
BaseController.prototype.req2params = req2paramsFn;
testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir);
});
function imageCompareFn(fixture, done) {
return function(err, tile) {
if (err) {
return done(err);
}
assert.imageEqualsFile(tile.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
};
}
it("should not use overview for tables without overviews", function(done){
testClient.getTile(testClient.defaultTableMapConfig('test_table'), 1, 0, 0,
imageCompareFn('test_table_1_0_0.png', done)
);
});
it("should not use overview for tables without overviews at z=2", function(done){
testClient.getTile(testClient.defaultTableMapConfig('test_table'), 2, 1, 1,
imageCompareFn('test_table_2_1_1.png', done)
);
});
it("should not use overview for tables without overviews at z=2", function(done){
testClient.getTile(testClient.defaultTableMapConfig('test_table'), 3, 3, 3,
imageCompareFn('test_table_3_3_3.png', done)
);
});
it("should use overview for zoom level 1", function(done){
testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 1, 0, 0,
imageCompareFn('_vovw_1_test_table_1_0_0.png', done)
);
});
it("should use overview for zoom level 1", function(done){
testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 2, 1, 1,
imageCompareFn('_vovw_2_test_table_2_1_1.png', done)
);
});
it("should not use overview for zoom level 3", function(done){
testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 3, 3, 3,
imageCompareFn('test_table_3_3_3.png', done)
);
});
});

7
test/acceptance/ported/support/ported_server_options.js
View File

@ -2,6 +2,10 @@ var _ = require('underscore');
var serverOptions = require('../../../../lib/cartodb/server_options');
var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token');
var mapnik = require('windshaft').mapnik;
var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
});
module.exports = _.extend({}, serverOptions, {
base_url: '/database/:dbname/table/:table',
@ -26,7 +30,8 @@ module.exports = _.extend({}, serverOptions, {
limits: {
render: 0,
cacheOnTimeout: true
}
},
queryRewriter: overviewsQueryRewriter
},
http: {
timeout: 5000,

BIN
test/fixtures/_vovw_1_test_table_1_0_0.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

BIN
test/fixtures/_vovw_2_test_table_2_1_1.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.4 KiB

BIN
test/fixtures/test_table_1_0_0.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 888 B

BIN
test/fixtures/test_table_2_1_1.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.0 KiB

BIN
test/fixtures/test_table_3_3_3.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 834 B

87
test/integration/mapconfig_overviews_adapter.js Normal file
View File

@ -0,0 +1,87 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter');
// configure redis pool instance to use in tests
var redisPool = new RedisPool(global.environment.redis);
var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool }));
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
describe('MapConfigOverviewsAdapter', function() {
it('should not modify layers for which no overviews are available', function(done) {
var sql = 'SELECT * FROM test_table';
var cartocss = '#layer { marker-fill: black; }';
var cartocss_version = '2.3.0';
var layer_without_overviews = {
type: 'cartodb',
options: {
sql: sql,
cartocss: cartocss,
cartocss_version: cartocss_version
}
};
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
assert.ok(!err);
assert.equal(layers.length, 1);
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, sql);
assert.equal(layers[0].options.cartocss, cartocss);
assert.equal(layers[0].options.cartocss_version, cartocss_version);
assert.equal(layers[0].options.overviews, undefined);
done();
});
});
});
describe('MapConfigOverviewsAdapter', function() {
it('should add overviews metadata for layers using tables with overviews', function(done) {
var sql = 'SELECT * FROM test_table_overviews';
var cartocss = '#layer { marker-fill: black; }';
var cartocss_version = '2.3.0';
var layer_without_overviews = {
type: 'cartodb',
options: {
sql: sql,
cartocss: cartocss,
cartocss_version: cartocss_version
}
};
mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
assert.ok(!err);
assert.equal(layers.length, 1);
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, sql);
assert.equal(layers[0].options.cartocss, cartocss);
assert.equal(layers[0].options.cartocss_version, cartocss_version);
assert.ok(layers[0].options.query_rewrite_data);
var expected_data = {
overviews: {
test_table_overviews: {
1: { table: '_vovw_1_test_table_overviews' },
2: { table: '_vovw_2_test_table_overviews' }
}
}
};
assert.deepEqual(layers[0].options.query_rewrite_data, expected_data);
done();
});
});
});

54
test/integration/overviews-metadata-api.js Normal file
View File

@ -0,0 +1,54 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
describe('OverviewsMetadataApi', function() {
var queryTablesApi, overviewsMetadataApi;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
queryTablesApi = new QueryTablesApi(pgQueryRunner);
overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
});
it('should return an empty relation for tables that have no overviews', function(done) {
var query = 'select * from test_table';
overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {});
done();
});
});
it('should return overviews metadata', function(done) {
var query = 'select * from test_table_overviews';
overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {
'test_table_overviews': {
1: { table: '_vovw_1_test_table_overviews' },
2: { table: '_vovw_2_test_table_overviews' }
}
});
done();
});
});
});

19
test/support/prepare_db.sh
View File

@ -79,19 +79,12 @@ if test x"$PREPARE_PGSQL" = xyes; then
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB}
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_TableMetadata.sql -o sql/CDB_TableMetadata.sql
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_ForeignTable.sql -o sql/CDB_ForeignTable.sql
cat sql/CDB_QueryStatements.sql |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
cat sql/CDB_QueryTables.sql |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
cat sql/CDB_TableMetadata.sql|
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
cat sql/CDB_ForeignTable.sql|sed -e 's/cartodb\./public./g' \
-e "s/''cartodb''/''public''/g" |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
for i in CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews
do
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
| psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
done
fi

14
test/support/sql/CDB_QueryStatements.sql
View File

@ -1,14 +0,0 @@
-- Return an array of statements found in the given query text
--
-- Regexp curtesy of Hubert Lubaczewski (depesz)
-- Implemented in plpython for performance reasons
--
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
RETURNS SETOF TEXT AS $$
import re
pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL )
for match in pat.findall(query):
cleaned = match[0].strip()
if ( cleaned ):
yield cleaned
$$ language 'plpythonu' IMMUTABLE STRICT;

78
test/support/sql/CDB_QueryTables.sql
View File

@ -1,78 +0,0 @@
-- Return an array of table names scanned by a given query
--
-- Requires PostgreSQL 9.x+
--
CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
RETURNS text[]
AS $$
DECLARE
exp XML;
tables text[];
rec RECORD;
rec2 RECORD;
BEGIN
tables := '{}';
FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
--RAISE WARNING 'Skipping %', rec.q;
CONTINUE;
END IF;
BEGIN
EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
EXCEPTION WHEN others THEN
-- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
-- the affected table ?
RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
RAISE EXCEPTION '%', SQLERRM;
CONTINUE;
END;
-- Now need to extract all values of <Relation-Name>
-- RAISE DEBUG 'Explain: %', exp;
FOR rec2 IN WITH
inp AS (
SELECT
xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
)
SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
LOOP
-- RAISE DEBUG 'tab: %', rec2.p;
-- RAISE DEBUG 'sc: %', rec2.sc;
tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
-- Remove duplicates and sort by name
IF array_upper(tables, 1) > 0 THEN
WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
SELECT array_agg(p) from dist into tables;
END IF;
--RAISE DEBUG 'Tables: %', tables;
return tables;
END
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
-- Keep CDB_QueryTables with same signature for backwards compatibility.
-- It should probably be removed in the future.
CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
RETURNS name[]
AS $$
BEGIN
RETURN CDB_QueryTablesText(query)::name[];
END
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

94
test/support/sql/windshaft.test.sql
View File

@ -238,3 +238,97 @@ CREATE INDEX test_big_poly_the_geom_webmercator_idx ON test_big_poly USING gist
GRANT ALL ON TABLE test_big_poly TO :TESTUSER;
GRANT SELECT ON TABLE test_big_poly TO :PUBLICUSER;
-- table with overviews
CREATE TABLE test_table_overviews (
updated_at timestamp without time zone DEFAULT now(),
created_at timestamp without time zone DEFAULT now(),
cartodb_id integer NOT NULL,
name character varying,
address character varying,
the_geom geometry,
the_geom_webmercator geometry,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE test_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER;
CREATE SEQUENCE test_table_overviews_cartodb_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE test_table_overviews_cartodb_id_seq OWNED BY test_table_overviews.cartodb_id;
SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true);
ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass);
INSERT INTO test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id);
CREATE INDEX test_table_overviews_the_geom_idx ON test_table_overviews USING gist (the_geom);
CREATE INDEX test_table_overviews_the_geom_webmercator_idx ON test_table_overviews USING gist (the_geom_webmercator);
GRANT ALL ON TABLE test_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER;
CREATE TABLE _vovw_1_test_table_overviews (
updated_at timestamp without time zone DEFAULT now(),
created_at timestamp without time zone DEFAULT now(),
cartodb_id integer NOT NULL,
name character varying,
address character varying,
the_geom geometry,
the_geom_webmercator geometry,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE _vovw_1_test_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE _vovw_1_test_table_overviews TO :PUBLICUSER;
CREATE TABLE _vovw_2_test_table_overviews (
updated_at timestamp without time zone DEFAULT now(),
created_at timestamp without time zone DEFAULT now(),
cartodb_id integer NOT NULL,
name character varying,
address character varying,
the_geom geometry,
the_geom_webmercator geometry,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER;
INSERT INTO _vovw_2_test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
INSERT INTO _vovw_1_test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241');

429
test/unit/cartodb/overviews_query_rewriter.js Normal file
View File

@ -0,0 +1,429 @@
require('../../support/test_helper');
var assert = require('assert');
var OverviewsQueryRewriter = require('../../../lib/cartodb/utils/overviews_query_rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
zoom_level: 'ZoomLevel()'
});
function normalize_whitespace(txt) {
return txt.replace(/\s+/g, " ").trim();
}
// compare SQL statements ignoring whitespace
function assertSameSql(sql1, sql2) {
assert.equal(normalize_whitespace(sql1), normalize_whitespace(sql2));
}
describe('Overviews query rewriter', function() {
it('does not alter queries if no overviews data is present', function(done){
var sql = "SELECT * FROM table1";
var overviews_sql = overviewsQueryRewriter.query(sql);
assert.equal(overviews_sql, sql);
overviews_sql = overviewsQueryRewriter.query(sql, {});
assert.equal(overviews_sql, sql);
overviews_sql = overviewsQueryRewriter.query(sql, { overviews: {} });
assert.equal(overviews_sql, sql);
done();
});
it('does not alter queries which don\'t use overviews', function(done){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table2: {
0: { table: 'table2_ov0' },
1: { table: 'table2_ov1' },
4: { table: 'table2_ov4' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
done();
});
// jshint multistr:true
it('generates query with single overview layer for level 0', function(done){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
)\
SELECT * FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query with single overview layer for level >0', function(done){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query with multiple overview layers for all levels up to N', function(done){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
1: { table: 'table1_ov1' },
2: { table: 'table1_ov2' },
3: { table: 'table1_ov3' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
UNION ALL\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\
UNION ALL\
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
)\
SELECT * FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query with multiple overview layers for random levels', function(done){
var sql = "SELECT * FROM table1";
var data = {
overviews: {
table1: {
0: { table: 'table1_ov0' },
1: { table: 'table1_ov1' },
6: { table: 'table1_ov6' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
UNION ALL\
SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
)\
SELECT * FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query for a table with explicit schema', function(done){
var sql = "SELECT * FROM public.table1";
var data = {
overviews: {
'public.table1': {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query for a table with explicit schema in the overviews info', function(done){
var sql = "SELECT * FROM public.table1";
var data = {
overviews: {
'public.table1': {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query for a table that needs quoting with explicit schema', function(done){
var sql = "SELECT * FROM public.\"table 1\"";
var data = {
overviews: {
'public."table 1"': {
2: { table: '"table 1_ov2"' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query for a table with explicit schema that needs quoting', function(done){
var sql = "SELECT * FROM \"user-1\".table1";
var data = {
overviews: {
'"user-1".table1': {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query for a table with explicit schema both needing quoting', function(done){
var sql = "SELECT * FROM \"user-1\".\"table 1\"";
var data = {
overviews: {
'"user-1"."table 1"': {
2: { table: '"table 1_ov2"' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with selected columns', function(done){
var sql = "SELECT column1, column2, column3 FROM table1";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT column1, column2, column3 FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with selected columns and all columns', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with a semicolon', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1;";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with extra whitespace', function(done){
var sql = " SELECT table1.* , column1,column2, column3 FROM table1 ";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('does not alter queries which have not the simple supported form', function(done){
var sql = "SELECT * FROM table1 WHERE column1='x'";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT * FROM table1 JOIN table2 ON (table1.col1=table2.col1)";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT a+b AS c FROM table1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT f(a) AS b FROM table1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT * FROM table1 AS x";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "WITH a AS (1) SELECT * FROM table1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT * FROM table1 WHERE a=1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "\
SELECT table1.* FROM table1 \
JOIN areas ON ST_Intersects(table1.the_geom, areas.the_geom) \
WHERE areas.name='A' \
";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
done();
});
});

60
test/unit/cartodb/table_name_parser.js Normal file
View File

@ -0,0 +1,60 @@
require('../../support/test_helper');
var assert = require('assert');
var TableNameParser = require('../../../lib/cartodb/utils/table_name_parser');
describe('TableNameParser', function() {
it('parses table names with scheme and quotes', function(done){
var test_cases = [
['xyz', { schema: null, table: 'xyz' }],
['"xyz"', { schema: null, table: 'xyz' }],
['"xy z"', { schema: null, table: 'xy z' }],
['"xy.z"', { schema: null, table: 'xy.z' }],
['"x.y.z"', { schema: null, table: 'x.y.z' }],
['abc.xyz', { schema: 'abc', table: 'xyz' }],
['"abc".xyz', { schema: 'abc', table: 'xyz' }],
['abc."xyz"', { schema: 'abc', table: 'xyz' }],
['"abc"."xyz"', { schema: 'abc', table: 'xyz' }],
['"a bc"."x yz"', { schema: 'a bc', table: 'x yz' }],
['"a bc".xyz', { schema: 'a bc', table: 'xyz' }],
['"a.bc".xyz', { schema: 'a.bc', table: 'xyz' }],
['"a.b.c".xyz', { schema: 'a.b.c', table: 'xyz' }],
['"a.b.c.".xyz', { schema: 'a.b.c.', table: 'xyz' }],
['"a""bc".xyz', { schema: 'a"bc', table: 'xyz' }],
['"a""bc"."x""yz"', { schema: 'a"bc', table: 'x"yz' }],
];
test_cases.forEach(function(test_case) {
var table_name = test_case[0];
var expected_result = test_case[1];
var result = TableNameParser.parse(table_name);
assert.deepEqual(result, expected_result);
});
done();
});
it('quotes identifiers that need quoting', function(done){
assert.equal(TableNameParser.quote('x yz'), '"x yz"');
assert.equal(TableNameParser.quote('x-yz'), '"x-yz"');
assert.equal(TableNameParser.quote('x.yz'), '"x.yz"');
done();
});
it('doubles quotes', function(done){
assert.equal(TableNameParser.quote('x"yz'), '"x""yz"');
assert.equal(TableNameParser.quote('x"y"z'), '"x""y""z"');
assert.equal(TableNameParser.quote('x""y"z'), '"x""""y""z"');
assert.equal(TableNameParser.quote('x "yz'), '"x ""yz"');
assert.equal(TableNameParser.quote('x"y-y"z'), '"x""y-y""z"');
done();
});
it('does not quote identifiers that don\'t need to be quoted', function(done){
assert.equal(TableNameParser.quote('xyz'), 'xyz');
assert.equal(TableNameParser.quote('x_z123'), 'x_z123');
done();
});
});