cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #388 from CartoDB/new_querytables_library

Browse Source 
Use new querytables library
This commit is contained in:
Raul Ochoa 2016-03-07 16:42:25 +01:00
commit 3cb007d147
24 changed files with 208 additions and 388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.travis.yml
View File

@ -12,6 +12,7 @@ addons:
before_install:
- npm install -g npm@2
- createdb template_postgis
- createuser publicuser
- psql -c "CREATE EXTENSION postgis" template_postgis
env:

96
lib/cartodb/api/query_tables_api.js
View File

@ -1,96 +0,0 @@
function QueryTablesApi(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
module.exports = QueryTablesApi;
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) {
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
// This is an Array, so no need to split into parts
var tableNames = rows[0].cdb_querytablestext;
return callback(null, tableNames);
});
};
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
var query = [
'WITH querytables AS (',
'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
')',
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m',
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
if (err || rows.length === 0) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables or last updated time: ' + msg));
return;
}
var result = rows[0];
// This is an Array, so no need to split into parts
var tableNames = result.tablenames;
var lastUpdatedTime = result.max || 0;
callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime * 1000
});
});
};
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, 0);
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
if (err) {
var msg = err.message ? err.message : err;
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var lastUpdated = 0;
if (rows.length !== 0) {
lastUpdated = rows[0].max || 0;
}
return callback(null, lastUpdated*1000);
});
};
function prepareSql(sql) {
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1')
;
}

10
lib/cartodb/api/tables_extent_api.js
View File

@ -13,13 +13,9 @@ module.exports = TablesExtentApi;
* `table_name` format as valid input
* @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
*/
TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) {
var estimatedExtentSQLs = tableNames.map(function(tableName) {
var schemaTable = tableName.split('.');
if (schemaTable.length > 1) {
return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')";
}
return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')";
TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
var estimatedExtentSQLs = tables.map(function(table) {
return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
});
var query = [

35
lib/cartodb/backends/pg_connection.js
View File

@ -1,5 +1,6 @@
var assert = require('assert');
var step = require('step');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
function PgConnection(metadataBackend) {
@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
}
);
};
/**
* Returns a `cartodb-psql` object for a given username.
* @param {String} username
* @param {Function} callback function({Error}, {PSQL})
*/
PgConnection.prototype.getConnection = function(username, callback) {
var self = this;
var params = {};
require('debug')('cachechan')("getConn1");
step(
function setAuth() {
self.setDBAuth(username, params, this);
},
function setConn(err) {
assert.ifError(err);
self.setDBConn(username, params, this);
},
function openConnection(err) {
assert.ifError(err);
return callback(err, new PSQL({
user: params.dbuser,
pass: params.dbpass,
host: params.dbhost,
port: params.dbport,
dbname: params.dbname
}));
}
);
};

24
lib/cartodb/cache/model/database_tables_entry.js vendored
View File

@ -1,24 +0,0 @@
var crypto = require('crypto');
function DatabaseTables(dbName, tableNames) {
this.namespace = 't';
this.dbName = dbName;
this.tableNames = tableNames;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tableNames.map(function(tableName) {
return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
}.bind(this));
};
DatabaseTables.prototype.getCacheChannel = function() {
return this.dbName + ':' + this.tableNames.join(',');
};
function shortHashKey(target) {
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
}

4
lib/cartodb/cache/named_map_provider_cache.js vendored
View File

@ -7,11 +7,10 @@ var queue = require('queue-async');
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
@ -30,7 +29,6 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
this.templateMaps,
this.pgConnection,
this.userLimitsApi,
this.queryTablesApi,
this.namedLayersAdapter,
user,
templateId,

36
lib/cartodb/controllers/layergroup.js
View File

@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var QueryTables = require('cartodb-query-tables');
/**
* @param {AuthApi} authApi
@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
* @param {WidgetBackend} widgetBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor
*/
function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
this.widgetBackend = widgetBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.layergroupAffectedTables = layergroupAffectedTables;
}
@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
global.logger.warn('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
self.surrogateKeysCache.tag(res, affectedTables);
}
self.send(req, res, body, status, headers);
}
@ -366,17 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
step(
function getConnection() {
self.pgConnection.getConnection(user, this);
},
function buildCacheChannel(err, tableNames) {
function getAffectedTables(err, connection) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
return tableNames;
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
function finish(err, affectedTables) {
callback(err, affectedTables);
}
this
);
},
function buildCacheChannel(err, tables) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables);
return tables;
},
callback
);
};

42
lib/cartodb/controllers/map.js
View File

@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var step = require('step');
var windshaft = require('windshaft');
var QueryTables = require('cartodb-query-tables');
var util = require('util');
var BaseController = require('./base');
@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
@ -26,7 +26,6 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
* @param {TemplateMaps} templateMaps
* @param {MapBackend} mapBackend
* @param metadataBackend
* @param {QueryTablesApi} queryTablesApi
* @param {OverviewsMetadataApi} overviewsMetadataApi
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
@ -34,7 +33,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
queryTablesApi, overviewsMetadataApi,
overviewsMetadataApi,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
@ -43,7 +42,6 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
this.metadataBackend = metadataBackend;
this.queryTablesApi = queryTablesApi;
this.overviewsMetadataApi = overviewsMetadataApi;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
@ -216,7 +214,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
self.queryTablesApi,
self.namedLayersAdapter,
cdbuser,
req.params.template_id,
@ -318,43 +315,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
var layergroupId = layergroup.layergroupid;
step(
function checkCachedAffectedTables() {
return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
function getPgConnection() {
self.pgConnection.getConnection(username, this);
},
function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
if (hasCache) {
var next = this;
var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
if (err) {
return next(err);
}
return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
});
} else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
}
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
// feed affected tables cache so it can be reused from, for instance, layergroup controller
self.layergroupAffectedTables.set(dbName, layergroupId, result);
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
// TODO this should take into account several URL patterns
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.set('Last-Modified', (new Date()).toUTCString());
res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
if (result.affectedTables && result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tableCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables && result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}

16
lib/cartodb/controllers/named_maps.js
View File

@ -9,8 +9,6 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
BaseController.call(this, authApi, pgConnection);
@ -44,7 +42,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
var self = this;
var dbName = req.params.dbname;
step(
function getAffectedTablesAndLastUpdatedTime() {
namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
@ -54,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
if (err) {
global.logger.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.affectedTables) {
if (!result || !!result.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.lastUpdatedTime);
lastModifiedDate = new Date(result.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
self.send(req, res, resource, 200);
@ -231,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
return next(null);
}
var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
var affectedTables = affectedTablesAndLastUpdate.tables || [];
if (affectedTables.length === 0) {
return next(null);

15
lib/cartodb/models/mapconfig/named_map_provider.js
View File

@ -5,17 +5,17 @@ var dot = require('dot');
var step = require('step');
var MapConfig = require('windshaft').model.MapConfig;
var templateName = require('../../backends/template_maps').templateName;
var QueryTables = require('cartodb-query-tables');
/**
* @constructor
* @type {NamedMapMapConfigProvider}
*/
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter,
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, namedLayersAdapter,
owner, templateId, config, authToken, params) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.namedLayersAdapter = namedLayersAdapter;
this.owner = owner;
@ -256,7 +256,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
},
function getAffectedTables(err, sql) {
assert.ifError(err);
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
step(
function getConnection() {
self.pgConnection.getConnection(self.owner, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;

6
lib/cartodb/server.js
View File

@ -19,7 +19,6 @@ var windshaft = require('windshaft');
var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api');
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
@ -52,7 +51,6 @@ module.exports = function(serverOptions) {
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var queryTablesApi = new QueryTablesApi(pgQueryRunner);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
@ -142,7 +140,7 @@ module.exports = function(serverOptions) {
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi);
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
});
@ -166,7 +164,6 @@ module.exports = function(serverOptions) {
new windshaft.backend.Widget(),
surrogateKeysCache,
userLimitsApi,
queryTablesApi,
layergroupAffectedTablesCache
).register(app);
@ -176,7 +173,6 @@ module.exports = function(serverOptions) {
templateMaps,
mapBackend,
metadataBackend,
queryTablesApi,
overviewsMetadataApi,
surrogateKeysCache,
userLimitsApi,

5
npm-shrinkwrap.json generated
View File

@ -470,6 +470,11 @@
"from": "lzma@>=1.3.7 <1.4.0",
"resolved": "https://registry.npmjs.org/lzma/-/lzma-1.3.7.tgz"
},
"cartodb-query-tables": {
"version": "0.1.0",
"from": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master",
"resolved": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
},
"node-statsd": {
"version": "0.0.7",
"from": "node-statsd@>=0.0.7 <0.1.0",

3
package.json
View File

@ -36,7 +36,8 @@
"redis-mpool": "~0.4.0",
"lru-cache": "2.6.5",
"lzma": "~1.3.7",
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
"cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
},
"devDependencies": {
"istanbul": "~0.3.6",

8
test/acceptance/multilayer.js
View File

@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var QueryTables = require('cartodb-query-tables');
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
@ -274,9 +274,9 @@ describe(suiteName, function() {
var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
'public.test_table',
'public.test_table_2'
helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
]).key().join(' '));

11
test/acceptance/multilayer_server.js
View File

@ -7,6 +7,7 @@ var _ = require('underscore');
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTables = require('cartodb-query-tables');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
@ -360,9 +361,11 @@ describe('tests from old api translated to multilayer', function() {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
var runQueryFn = PgQueryRunner.prototype.run;
PgQueryRunner.prototype.run = function(username, query, callback) {
return callback(new Error('failed to query database for affected tables'), []);
var affectedFn = QueryTables.getAffectedTablesFromQuery;
QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
affectedFn({query: function(query, callback) {
return callback(new Error('fake error message'), []);
}}, username, query, callback);
};
// reset internal cacheChannel cache
@ -387,7 +390,7 @@ describe('tests from old api translated to multilayer', function() {
},
function(res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
PgQueryRunner.prototype.run = runQueryFn;
QueryTables.getAffectedTablesFromQuery = affectedFn;
done();
}
);

8
test/acceptance/templates.js
View File

@ -3,8 +3,8 @@ var _ = require('underscore');
var redis = require('redis');
var step = require('step');
var strftime = require('strftime');
var QueryTables = require('cartodb-query-tables');
var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var redis_stats_db = 5;
// Pollute the PG environment to make sure
@ -1405,7 +1405,8 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');
helper.checkSurrogateKey(res, expectedSurrogateKey);
@ -1488,7 +1489,8 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');
helper.checkSurrogateKey(res, expectedSurrogateKey);

4
test/integration/overviews-metadata-api.js
View File

@ -7,20 +7,18 @@ var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
describe('OverviewsMetadataApi', function() {
var queryTablesApi, overviewsMetadataApi;
var overviewsMetadataApi;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
queryTablesApi = new QueryTablesApi(pgQueryRunner);
overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
});

55
test/integration/query-tables-api.js
View File

@ -1,55 +0,0 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
describe('QueryTablesApi', function() {
var queryTablesApi;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
queryTablesApi = new QueryTablesApi(pgQueryRunner);
});
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
it('should return an object with affected tables array and last updated time', function(done) {
var query = 'select * from test_table';
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {
affectedTables: [ 'public.test_table' ],
lastUpdatedTime: 1234567890123
});
done();
});
});
it('should work with private tables', function(done) {
var query = 'select * from test_table_private_1';
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {
affectedTables: [ 'public.test_table_private_1' ],
lastUpdatedTime: 1234567890123
});
done();
});
});
});

71
test/integration/query-tables.js Normal file
View File

@ -0,0 +1,71 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var QueryTables = require('cartodb-query-tables');
describe('QueryTables', function() {
var connection;
before(function(done) {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
pgConnection.getConnection('localhost', function(err, pgConnection) {
if (err) {
return done(err);
}
connection = pgConnection;
return done();
});
});
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
it('should return an object with affected tables array and last updated time', function(done) {
var query = 'select * from test_table';
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
assert.ok(!err, err);
assert.equal(result.getLastUpdatedAt(), 1234567890123);
assert.equal(result.tables.length, 1);
assert.deepEqual(result.tables[0], {
dbname: 'test_windshaft_cartodb_user_1_db',
schema_name: 'public',
table_name: 'test_table',
updated_at: new Date(1234567890123)
});
done();
});
});
it('should work with private tables', function(done) {
var query = 'select * from test_table_private_1';
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
assert.ok(!err, err);
assert.equal(result.getLastUpdatedAt(), 1234567890123);
assert.equal(result.tables.length, 1);
assert.deepEqual(result.tables[0], {
dbname: 'test_windshaft_cartodb_user_1_db',
schema_name: 'public',
table_name: 'test_table_private_1',
updated_at: new Date(1234567890123)
});
done();
});
});
});

12
test/support/prepare_db.sh
View File

@ -78,9 +78,15 @@ if test x"$PREPARE_PGSQL" = xyes; then
sed "s/:TESTPASS/${TESTPASS}/" |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql sql/CDB_Overviews.sql |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews'
for i in ${SQL_SCRIPTS}
do
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
| psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
done
fi

46
test/support/sql/CDB_Overviews.sql
View File

@ -1,46 +0,0 @@
-- Mockup for CDB_Overviews
CREATE OR REPLACE FUNCTION CDB_Overviews(table_names regclass[])
RETURNS TABLE(base_table regclass, z integer, overview_table regclass)
AS $$
BEGIN
IF (SELECT 'test_table_overviews'::regclass = ANY (table_names)) THEN
RETURN QUERY
SELECT 'test_table_overviews'::regclass AS base_table, 1 AS z, '_vovw_1_test_table_overviews'::regclass AS overview_table
UNION ALL
SELECT 'test_table_overviews'::regclass AS base_table, 2 AS z, '_vovw_2_test_table_overviews'::regclass AS overview_table;
ELSE
RETURN;
END IF;
END
$$ LANGUAGE PLPGSQL;
CREATE OR REPLACE FUNCTION CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
BEGIN
CASE
WHEN scaleDenominator > 500000000 THEN RETURN 0;
WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
WHEN scaleDenominator <= 100 THEN RETURN 23;
END CASE;
END
$$ LANGUAGE plpgsql IMMUTABLE;

78
test/support/sql/CDB_QueryTables.sql
View File

@ -1,78 +0,0 @@
-- Return an array of table names scanned by a given query
--
-- Requires PostgreSQL 9.x+
--
CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
RETURNS text[]
AS $$
DECLARE
exp XML;
tables text[];
rec RECORD;
rec2 RECORD;
BEGIN
tables := '{}';
FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
--RAISE WARNING 'Skipping %', rec.q;
CONTINUE;
END IF;
BEGIN
EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
EXCEPTION WHEN others THEN
-- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
-- the affected table ?
RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
RAISE EXCEPTION '%', SQLERRM;
CONTINUE;
END;
-- Now need to extract all values of <Relation-Name>
-- RAISE DEBUG 'Explain: %', exp;
FOR rec2 IN WITH
inp AS (
SELECT
xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
)
SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
LOOP
-- RAISE DEBUG 'tab: %', rec2.p;
-- RAISE DEBUG 'sc: %', rec2.sc;
tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
-- Remove duplicates and sort by name
IF array_upper(tables, 1) > 0 THEN
WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
SELECT array_agg(p) from dist into tables;
END IF;
--RAISE DEBUG 'Tables: %', tables;
return tables;
END
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
-- Keep CDB_QueryTables with same signature for backwards compatibility.
-- It should probably be removed in the future.
CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
RETURNS name[]
AS $$
BEGIN
RETURN CDB_QueryTablesText(query)::name[];
END
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

0
test/support/sql/CDB_QueryStatements.sql → test/support/sql/_CDB_QueryStatements.sql
View File

10
test/support/test_helper.js
View File

@ -63,7 +63,15 @@ function checkCache(res) {
function checkSurrogateKey(res, expectedKey) {
assert.ok(res.headers.hasOwnProperty('surrogate-key'));
assert.equal(res.headers['surrogate-key'], expectedKey);
function createSet(keys, key) {
keys[key] = true;
return keys;
}
var keys = res.headers['surrogate-key'].split(' ').reduce(createSet, {});
var expectedKeys = expectedKey.split(' ').reduce(createSet, {});
assert.deepEqual(keys, expectedKeys);
}
var redisClient;