cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'new_querytables_library' into analysis-layers

Browse Source
This commit is contained in:
Raul Ochoa 2016-03-04 12:08:57 +01:00
commit c7bdabfc65
23 changed files with 325 additions and 384 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.travis.yml
View File

@ -1,11 +1,15 @@
sudo: true
sudo: false
addons:
postgresql: "9.3"
apt:
packages:
- pkg-config
- libcairo2-dev
- libjpeg8-dev
- libgif-dev
before_install:
- sudo apt-get update
- sudo apt-get install -y postgresql-plpython-9.3 pkg-config libcairo2-dev libjpeg8-dev libgif-dev
- npm install -g npm@2
- createdb template_postgis
- createuser publicuser

10
NEWS.md
View File

@ -1,8 +1,16 @@
# Changelog
## 2.26.4
Released 2016-mm-dd
## 2.26.3
Released 2016-mm-dd
Released 2016-03-03
Improvements:
- Optimize overviews queries for efficient spatial filtering in PostgreSQL
## 2.26.2

90
lib/cartodb/api/query_tables_api.js
View File

@ -1,90 +0,0 @@
function QueryTablesApi(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
module.exports = QueryTablesApi;
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) {
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
// This is an Array, so no need to split into parts
var tableNames = rows[0].cdb_querytablestext;
return callback(null, tableNames);
});
};
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
var query =
'SELECT * FROM CDB_QueryTables_Updated_At($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
if (err) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables or last updated time: ' + msg));
return;
}
var affectedTables = rows;
var updatedTimes = affectedTables.map(function getUpdateDate(table) {
return table.updated_at;
});
var lastUpdatedTime = (affectedTables.length === 0 ? 0 : Math.max.apply(null, updatedTimes)) || 0;
callback(null, {
affectedTables: affectedTables,
lastUpdatedTime: lastUpdatedTime
});
});
};
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, 0);
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
if (err) {
var msg = err.message ? err.message : err;
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var lastUpdated = 0;
if (rows.length !== 0) {
lastUpdated = rows[0].max || 0;
}
return callback(null, lastUpdated*1000);
});
};
function prepareSql(sql) {
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1')
;
}

35
lib/cartodb/backends/pg_connection.js
View File

@ -1,5 +1,6 @@
var assert = require('assert');
var step = require('step');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
function PgConnection(metadataBackend) {
@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
}
);
};
/**
* Returns a `cartodb-psql` object for a given username.
* @param {String} username
* @param {Function} callback function({Error}, {PSQL})
*/
PgConnection.prototype.getConnection = function(username, callback) {
var self = this;
var params = {};
require('debug')('cachechan')("getConn1");
step(
function setAuth() {
self.setDBAuth(username, params, this);
},
function setConn(err) {
assert.ifError(err);
self.setDBConn(username, params, this);
},
function openConnection(err) {
assert.ifError(err);
return callback(err, new PSQL({
user: params.dbuser,
pass: params.dbpass,
host: params.dbhost,
port: params.dbport,
dbname: params.dbname
}));
}
);
};

26
lib/cartodb/cache/model/database_tables_entry.js vendored
View File

@ -1,26 +0,0 @@
var crypto = require('crypto');
function DatabaseTables(tables) {
this.namespace = 't';
this.tables = tables;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tables.map(function(table) {
return this.namespace + ':' + shortHashKey(table.dbname + ':' + table.table_name + '.' + table.schema_name);
}.bind(this));
};
DatabaseTables.prototype.getCacheChannel = function() {
var key = this.tables.map(function(table) {
return table.dbname + ':' + table.schema_name + "." + table.table_name;
}).join(";;");
return key;
};
function shortHashKey(target) {
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
}

4
lib/cartodb/cache/named_map_provider_cache.js vendored
View File

@ -7,11 +7,10 @@ var queue = require('queue-async');
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
@ -30,7 +29,6 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
this.templateMaps,
this.pgConnection,
this.userLimitsApi,
this.queryTablesApi,
this.namedLayersAdapter,
user,
templateId,

35
lib/cartodb/controllers/layergroup.js
View File

@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var QueryTables = require('cartodb-query-tables');
/**
* @param {AuthApi} authApi
@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
* @param {WidgetBackend} widgetBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor
*/
function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
this.widgetBackend = widgetBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.layergroupAffectedTables = layergroupAffectedTables;
}
@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
global.logger.warn('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
self.surrogateKeysCache.tag(res, affectedTables);
}
self.send(req, res, body, status, headers);
}
@ -366,20 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel
step(
function getConnection() {
self.pgConnection.getConnection(user, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function buildCacheChannel(err, tables) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, tables);
return tables;
},
function finish(err, tables) {
if(tables === undefined){
callback(err);
}else{
callback(err, tables.affectedTables);
}
}
callback
);
};

42
lib/cartodb/controllers/map.js
View File

@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var step = require('step');
var windshaft = require('windshaft');
var QueryTables = require('cartodb-query-tables');
var util = require('util');
var BaseController = require('./base');
@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var MapConfigAnalysisLayersAdapter = require('../models/mapconfig_analysis_layers_adapter');
@ -27,7 +27,6 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
* @param {TemplateMaps} templateMaps
* @param {MapBackend} mapBackend
* @param metadataBackend
* @param {QueryTablesApi} queryTablesApi
* @param {OverviewsMetadataApi} overviewsMetadataApi
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
@ -35,7 +34,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
queryTablesApi, overviewsMetadataApi,
overviewsMetadataApi,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
@ -44,7 +43,6 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
this.metadataBackend = metadataBackend;
this.queryTablesApi = queryTablesApi;
this.overviewsMetadataApi = overviewsMetadataApi;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
@ -237,7 +235,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
self.queryTablesApi,
self.namedLayersAdapter,
cdbuser,
req.params.template_id,
@ -339,43 +336,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
var layergroupId = layergroup.layergroupid;
step(
function checkCachedAffectedTables() {
return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
function getPgConnection() {
self.pgConnection.getConnection(username, this);
},
function getAffectedTablesAndLastUpdatedTime(err) {
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
// if (hasCache) {
// var next = this;
// var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
// self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
// if (err) {
// return next(err);
// }
// return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
// });
// } else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
//}
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
// feed affected tables cache so it can be reused from, for instance, layergroup controller
self.layergroupAffectedTables.set(dbName, layergroupId, result);
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
// TODO this should take into account several URL patterns
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.set('Last-Modified', (new Date()).toUTCString());
res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
if (result.affectedTables && result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tableCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables && result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}

15
lib/cartodb/controllers/named_maps.js
View File

@ -9,8 +9,6 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
BaseController.call(this, authApi, pgConnection);
@ -53,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
if (err) {
global.logger.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.affectedTables) {
if (!result || !!result.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.lastUpdatedTime);
lastModifiedDate = new Date(result.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(result.affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
self.send(req, res, resource, 200);
@ -230,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
return next(null);
}
var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
var affectedTables = affectedTablesAndLastUpdate.tables || [];
if (affectedTables.length === 0) {
return next(null);

15
lib/cartodb/models/mapconfig/named_map_provider.js
View File

@ -5,17 +5,17 @@ var dot = require('dot');
var step = require('step');
var MapConfig = require('windshaft').model.MapConfig;
var templateName = require('../../backends/template_maps').templateName;
var QueryTables = require('cartodb-query-tables');
/**
* @constructor
* @type {NamedMapMapConfigProvider}
*/
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter,
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, namedLayersAdapter,
owner, templateId, config, authToken, params) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.namedLayersAdapter = namedLayersAdapter;
this.owner = owner;
@ -256,7 +256,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
},
function getAffectedTables(err, sql) {
assert.ifError(err);
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
step(
function getConnection() {
self.pgConnection.getConnection(self.owner, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;

6
lib/cartodb/server.js
View File

@ -19,7 +19,6 @@ var windshaft = require('windshaft');
var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api');
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
@ -52,7 +51,6 @@ module.exports = function(serverOptions) {
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var queryTablesApi = new QueryTablesApi(pgQueryRunner);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
@ -142,7 +140,7 @@ module.exports = function(serverOptions) {
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi);
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
});
@ -166,7 +164,6 @@ module.exports = function(serverOptions) {
new windshaft.backend.Widget(),
surrogateKeysCache,
userLimitsApi,
queryTablesApi,
layergroupAffectedTablesCache
).register(app);
@ -176,7 +173,6 @@ module.exports = function(serverOptions) {
templateMaps,
mapBackend,
metadataBackend,
queryTablesApi,
overviewsMetadataApi,
surrogateKeysCache,
userLimitsApi,

14
lib/cartodb/utils/overviews_query_rewriter.js
View File

@ -74,14 +74,9 @@ function overviews_view_name(table) {
}
// replace a table name in a query by anoter name
function replace_table_in_query(sql, old_table_name, new_table_name) {
function replace_table_in_query(sql, old_table_name, replacement) {
var old_table = TableNameParser.parse(old_table_name);
var new_table = TableNameParser.parse(new_table_name);
var old_table_ident = TableNameParser.table_identifier(old_table);
var new_table_ident = TableNameParser.table_identifier(new_table);
// text that will be substituted by the table name pattern
var replacement = new_table_ident;
// regular expression prefix (beginning) to match a table name
function pattern_prefix(schema, identifier) {
@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) {
function overviews_query(query, overviews, zoom_level_expression) {
var replaced_query = query;
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
var replacement;
for ( var table in overviews ) {
if (overviews.hasOwnProperty(table)) {
var table_overviews = overviews[table];
var table_view = overviews_view_name(table);
replaced_query = replace_table_in_query(replaced_query, table, table_view);
sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )";
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
replaced_query = replace_table_in_query(replaced_query, table, replacement);
}
}
if ( replaced_query !== query ) {
@ -173,7 +169,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data) {
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
return !!sql.match(
/^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
/^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
);
};

110
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "2.26.3",
"version": "2.26.4",
"dependencies": {
"body-parser": {
"version": "1.14.2",
@ -84,14 +84,14 @@
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
},
"mime-types": {
"version": "2.1.10",
"version": "2.1.9",
"from": "mime-types@>=2.1.9 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
"version": "1.22.0",
"from": "mime-db@>=1.22.0 <1.23.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz"
"version": "1.21.0",
"from": "mime-db@>=1.21.0 <1.22.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz"
}
}
}
@ -594,14 +594,14 @@
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
"dependencies": {
"mime-types": {
"version": "2.1.10",
"version": "2.1.9",
"from": "mime-types@>=2.1.6 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
"version": "1.22.0",
"from": "mime-db@>=1.22.0 <1.23.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz"
"version": "1.21.0",
"from": "mime-db@>=1.21.0 <1.22.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz"
}
}
},
@ -659,7 +659,7 @@
"dependencies": {
"unpipe": {
"version": "1.0.0",
"from": "unpipe@1.0.0",
"from": "unpipe@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
}
}
@ -774,7 +774,7 @@
},
"type-is": {
"version": "1.6.11",
"from": "type-is@>=1.6.10 <1.7.0",
"from": "type-is@>=1.6.6 <1.7.0",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.11.tgz",
"dependencies": {
"media-typer": {
@ -783,14 +783,14 @@
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
},
"mime-types": {
"version": "2.1.10",
"version": "2.1.9",
"from": "mime-types@>=2.1.6 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
"version": "1.22.0",
"from": "mime-db@>=1.22.0 <1.23.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz"
"version": "1.21.0",
"from": "mime-db@>=1.21.0 <1.22.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz"
}
}
}
@ -872,6 +872,11 @@
"from": "lzma@>=1.3.7 <1.4.0",
"resolved": "https://registry.npmjs.org/lzma/-/lzma-1.3.7.tgz"
},
"cartodb-query-tables": {
"version": "0.1.0",
"from": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master",
"resolved": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
},
"node-statsd": {
"version": "0.0.7",
"from": "node-statsd@>=0.0.7 <0.1.0",
@ -922,9 +927,9 @@
"resolved": "https://registry.npmjs.org/request/-/request-2.62.0.tgz",
"dependencies": {
"bl": {
"version": "1.0.3",
"version": "1.0.2",
"from": "bl@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz",
"resolved": "https://registry.npmjs.org/bl/-/bl-1.0.2.tgz",
"dependencies": {
"readable-stream": {
"version": "2.0.5",
@ -987,7 +992,7 @@
"dependencies": {
"async": {
"version": "1.5.2",
"from": "async@>=1.0.0 <2.0.0",
"from": "async@>=1.4.0 <2.0.0",
"resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
}
}
@ -998,14 +1003,14 @@
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
},
"mime-types": {
"version": "2.1.10",
"from": "mime-types@>=2.1.9 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz",
"version": "2.1.9",
"from": "mime-types@>=2.1.2 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
"version": "1.22.0",
"from": "mime-db@>=1.22.0 <1.23.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz"
"version": "1.21.0",
"from": "mime-db@>=1.21.0 <1.22.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz"
}
}
},
@ -1126,21 +1131,14 @@
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz",
"dependencies": {
"ansi-styles": {
"version": "2.2.0",
"version": "2.1.0",
"from": "ansi-styles@>=2.1.0 <3.0.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.0.tgz",
"dependencies": {
"color-convert": {
"version": "1.0.0",
"from": "color-convert@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.0.0.tgz"
}
}
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz"
},
"escape-string-regexp": {
"version": "1.0.5",
"version": "1.0.4",
"from": "escape-string-regexp@>=1.0.2 <2.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.4.tgz"
},
"has-ansi": {
"version": "2.0.0",
@ -1155,9 +1153,9 @@
}
},
"strip-ansi": {
"version": "3.0.1",
"version": "3.0.0",
"from": "strip-ansi@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz",
"dependencies": {
"ansi-regex": {
"version": "2.0.0",
@ -1186,9 +1184,9 @@
}
},
"is-my-json-valid": {
"version": "2.13.1",
"version": "2.12.4",
"from": "is-my-json-valid@>=2.12.0 <3.0.0",
"resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz",
"resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.4.tgz",
"dependencies": {
"generate-function": {
"version": "2.0.0",
@ -4134,6 +4132,7 @@
"turbo-cartocss": {
"version": "0.4.0",
"from": "turbo-cartocss@>=0.4.0 <0.5.0",
"resolved": "https://registry.npmjs.org/turbo-cartocss/-/turbo-cartocss-0.4.0.tgz",
"dependencies": {
"browser-request": {
"version": "0.3.3",
@ -4221,9 +4220,9 @@
}
},
"postcss": {
"version": "5.0.16",
"version": "5.0.19",
"from": "postcss@>=5.0.13 <6.0.0",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-5.0.16.tgz",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-5.0.19.tgz",
"dependencies": {
"supports-color": {
"version": "3.1.2",
@ -4270,9 +4269,28 @@
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
},
"aws4": {
"version": "1.2.1",
"version": "1.3.2",
"from": "aws4@>=1.2.1 <2.0.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.2.1.tgz"
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz",
"dependencies": {
"lru-cache": {
"version": "4.0.0",
"from": "lru-cache@>=4.0.0 <5.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz",
"dependencies": {
"pseudomap": {
"version": "1.0.2",
"from": "pseudomap@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz"
},
"yallist": {
"version": "2.0.0",
"from": "yallist@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.0.0.tgz"
}
}
}
}
},
"bl": {
"version": "1.0.3",

5
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "2.26.3",
"version": "2.26.4",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@ -37,7 +37,8 @@
"redis-mpool": "~0.4.0",
"lru-cache": "2.6.5",
"lzma": "~1.3.7",
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
"cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
},
"devDependencies": {
"istanbul": "~0.3.6",

4
test/acceptance/multilayer.js
View File

@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var QueryTables = require('cartodb-query-tables');
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
@ -274,7 +274,7 @@ describe(suiteName, function() {
var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry([
helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
]).key().join(' '));

11
test/acceptance/multilayer_server.js
View File

@ -7,6 +7,7 @@ var _ = require('underscore');
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTables = require('cartodb-query-tables');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
@ -360,9 +361,11 @@ describe('tests from old api translated to multilayer', function() {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
var runQueryFn = PgQueryRunner.prototype.run;
PgQueryRunner.prototype.run = function(username, query, callback) {
return callback(new Error('failed to query database for affected tables'), []);
var affectedFn = QueryTables.getAffectedTablesFromQuery;
QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
affectedFn({query: function(query, callback) {
return callback(new Error('fake error message'), []);
}}, username, query, callback);
};
// reset internal cacheChannel cache
@ -387,7 +390,7 @@ describe('tests from old api translated to multilayer', function() {
},
function(res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
PgQueryRunner.prototype.run = runQueryFn;
QueryTables.getAffectedTablesFromQuery = affectedFn;
done();
}
);

6
test/acceptance/templates.js
View File

@ -3,8 +3,8 @@ var _ = require('underscore');
var redis = require('redis');
var step = require('step');
var strftime = require('strftime');
var QueryTables = require('cartodb-query-tables');
var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var redis_stats_db = 5;
// Pollute the PG environment to make sure
@ -1405,7 +1405,7 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');
@ -1489,7 +1489,7 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');

4
test/integration/overviews-metadata-api.js
View File

@ -7,20 +7,18 @@ var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
describe('OverviewsMetadataApi', function() {
var queryTablesApi, overviewsMetadataApi;
var overviewsMetadataApi;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
queryTablesApi = new QueryTablesApi(pgQueryRunner);
overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
});

56
test/integration/query-tables-api.js
View File

@ -1,56 +0,0 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
describe('QueryTablesApi', function() {
var queryTablesApi;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
queryTablesApi = new QueryTablesApi(pgQueryRunner);
});
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
it('should return an object with affected tables array and last updated time', function(done) {
var query = 'select * from test_table';
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {
affectedTables: [{dbname: "test_windshaft_cartodb_user_1_db", schema_name: "public",
"table_name": 'test_table', updated_at: new Date(1234567890123)}],
lastUpdatedTime: 1234567890123
});
done();
});
});
it('should work with private tables', function(done) {
var query = 'select * from test_table_private_1';
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {
affectedTables: [{dbname: "test_windshaft_cartodb_user_1_db", schema_name: "public",
"table_name": 'test_table_private_1', updated_at: new Date(1234567890123)}],
lastUpdatedTime: 1234567890123
});
done();
});
});
});

71
test/integration/query-tables.js Normal file
View File

@ -0,0 +1,71 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var QueryTables = require('cartodb-query-tables');
describe('QueryTables', function() {
var connection;
before(function(done) {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
pgConnection.getConnection('localhost', function(err, pgConnection) {
if (err) {
return done(err);
}
connection = pgConnection;
return done();
});
});
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
it('should return an object with affected tables array and last updated time', function(done) {
var query = 'select * from test_table';
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
assert.ok(!err, err);
assert.equal(result.getLastUpdatedAt(), 1234567890123);
assert.equal(result.tables.length, 1);
assert.deepEqual(result.tables[0], {
dbname: 'test_windshaft_cartodb_user_1_db',
schema_name: 'public',
table_name: 'test_table',
updated_at: new Date(1234567890123)
});
done();
});
});
it('should work with private tables', function(done) {
var query = 'select * from test_table_private_1';
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
assert.ok(!err, err);
assert.equal(result.getLastUpdatedAt(), 1234567890123);
assert.equal(result.tables.length, 1);
assert.deepEqual(result.tables[0], {
dbname: 'test_windshaft_cartodb_user_1_db',
schema_name: 'public',
table_name: 'test_table_private_1',
updated_at: new Date(1234567890123)
});
done();
});
});
});

6
test/support/prepare_db.sh
View File

@ -78,8 +78,10 @@ if test x"$PREPARE_PGSQL" = xyes; then
sed "s/:TESTPASS/${TESTPASS}/" |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB}
for i in CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews
cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews'
for i in ${SQL_SCRIPTS}
do
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \

16
test/support/sql/_CDB_QueryStatements.sql Normal file
View File

@ -0,0 +1,16 @@
-- DUMMY IMPLEMENTATION
-- Ref: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_QueryStatements.sql
-- Originally implemented in plpython for performance reasons
-- Return an array of statements found in the given query text
--
-- Regexp curtesy of Hubert Lubaczewski (depesz)
--
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
RETURNS SETOF TEXT AS $$
with matches as (
select regexp_matches($1, $regexp$((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)$regexp$, 'g') as m
)
select btrim(m[1]) from matches
$$
LANGUAGE SQL IMMUTABLE STRICT;

118
test/unit/cartodb/overviews_query_rewriter.js
View File

@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() {
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
) AS \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
) AS \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT column1, column2, column3 FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with selected columns and all columns', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with a semicolon', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1;";
var sql = "SELECT column1, column2, column3 FROM table1;";
var data = {
overviews: {
table1: {
@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() {
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
) AS _vovw_table1;\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with extra whitespace', function(done){
var sql = " SELECT table1.* , column1,column2, column3 FROM table1 ";
var sql = " SELECT column1,column2, column3 FROM table1 ";
var data = {
overviews: {
table1: {
@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1,column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -422,6 +388,10 @@ describe('Overviews query rewriter', function() {
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT table1.*, column1, column2, column3 FROM table1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
done();
});