diff --git a/.travis.yml b/.travis.yml index 3d9796bc..3be5a067 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ addons: before_install: - npm install -g npm@2 - createdb template_postgis + - createuser publicuser - psql -c "CREATE EXTENSION postgis" template_postgis env: diff --git a/lib/cartodb/api/query_tables_api.js b/lib/cartodb/api/query_tables_api.js deleted file mode 100644 index af2fc978..00000000 --- a/lib/cartodb/api/query_tables_api.js +++ /dev/null @@ -1,96 +0,0 @@ -function QueryTablesApi(pgQueryRunner) { - this.pgQueryRunner = pgQueryRunner; -} - -var affectedTableRegexCache = { - bbox: /!bbox!/g, - scale_denominator: /!scale_denominator!/g, - pixel_width: /!pixel_width!/g, - pixel_height: /!pixel_height!/g -}; - -module.exports = QueryTablesApi; - - -QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) { - var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)'; - - this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) { - if (err){ - var msg = err.message ? err.message : err; - callback(new Error('could not fetch source tables: ' + msg)); - return; - } - - // This is an Array, so no need to split into parts - var tableNames = rows[0].cdb_querytablestext; - return callback(null, tableNames); - }); -}; - -QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) { - var query = [ - 'WITH querytables AS (', - 'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames', - ')', - 'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max', - 'FROM CDB_TableMetadata m', - 'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])' - ].join(' '); - - this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) { - if (err || rows.length === 0) { - var msg = err.message ? err.message : err; - callback(new Error('could not fetch affected tables or last updated time: ' + msg)); - return; - } - - var result = rows[0]; - - // This is an Array, so no need to split into parts - var tableNames = result.tablenames; - - var lastUpdatedTime = result.max || 0; - - callback(null, { - affectedTables: tableNames, - lastUpdatedTime: lastUpdatedTime * 1000 - }); - }); -}; - -QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) { - if (!Array.isArray(tableNames) || tableNames.length === 0) { - return callback(null, 0); - } - - var query = [ - 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max', - 'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[', - tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','), - '])' - ].join(' '); - - this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) { - if (err) { - var msg = err.message ? err.message : err; - return callback(new Error('could not fetch affected tables or last updated time: ' + msg)); - } - // when the table has not updated_at means it hasn't been changed so a default last_updated is set - var lastUpdated = 0; - if (rows.length !== 0) { - lastUpdated = rows[0].max || 0; - } - - return callback(null, lastUpdated*1000); - }); -}; - -function prepareSql(sql) { - return sql - .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') - .replace(affectedTableRegexCache.scale_denominator, '0') - .replace(affectedTableRegexCache.pixel_width, '1') - .replace(affectedTableRegexCache.pixel_height, '1') - ; -} diff --git a/lib/cartodb/api/tables_extent_api.js b/lib/cartodb/api/tables_extent_api.js index d4293ed7..7b534a3f 100644 --- a/lib/cartodb/api/tables_extent_api.js +++ b/lib/cartodb/api/tables_extent_api.js @@ -13,13 +13,9 @@ module.exports = TablesExtentApi; * `table_name` format as valid input * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north` */ -TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) { - var estimatedExtentSQLs = tableNames.map(function(tableName) { - var schemaTable = tableName.split('.'); - if (schemaTable.length > 1) { - return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')"; - } - return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')"; +TablesExtentApi.prototype.getBounds = function (username, tables, callback) { + var estimatedExtentSQLs = tables.map(function(table) { + return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')"; }); var query = [ diff --git a/lib/cartodb/backends/pg_connection.js b/lib/cartodb/backends/pg_connection.js index 2ca56f5b..d98bb703 100644 --- a/lib/cartodb/backends/pg_connection.js +++ b/lib/cartodb/backends/pg_connection.js @@ -1,5 +1,6 @@ var assert = require('assert'); var step = require('step'); +var PSQL = require('cartodb-psql'); var _ = require('underscore'); function PgConnection(metadataBackend) { @@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) { } ); }; + + +/** + * Returns a `cartodb-psql` object for a given username. + * @param {String} username + * @param {Function} callback function({Error}, {PSQL}) + */ + +PgConnection.prototype.getConnection = function(username, callback) { + var self = this; + + var params = {}; + + require('debug')('cachechan')("getConn1"); + step( + function setAuth() { + self.setDBAuth(username, params, this); + }, + function setConn(err) { + assert.ifError(err); + self.setDBConn(username, params, this); + }, + function openConnection(err) { + assert.ifError(err); + return callback(err, new PSQL({ + user: params.dbuser, + pass: params.dbpass, + host: params.dbhost, + port: params.dbport, + dbname: params.dbname + })); + } + ); +}; diff --git a/lib/cartodb/cache/model/database_tables_entry.js b/lib/cartodb/cache/model/database_tables_entry.js deleted file mode 100644 index 4d269137..00000000 --- a/lib/cartodb/cache/model/database_tables_entry.js +++ /dev/null @@ -1,24 +0,0 @@ -var crypto = require('crypto'); - -function DatabaseTables(dbName, tableNames) { - this.namespace = 't'; - this.dbName = dbName; - this.tableNames = tableNames; -} - -module.exports = DatabaseTables; - - -DatabaseTables.prototype.key = function() { - return this.tableNames.map(function(tableName) { - return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName); - }.bind(this)); -}; - -DatabaseTables.prototype.getCacheChannel = function() { - return this.dbName + ':' + this.tableNames.join(','); -}; - -function shortHashKey(target) { - return crypto.createHash('sha256').update(target).digest('base64').substring(0,6); -} diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index 21c33142..0048cb63 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -7,11 +7,10 @@ var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) { +function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.userLimitsApi = userLimitsApi; - this.queryTablesApi = queryTablesApi; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); @@ -30,7 +29,6 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.templateMaps, this.pgConnection, this.userLimitsApi, - this.queryTablesApi, this.namedLayersAdapter, user, templateId, diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 0a7e7698..b6b4d640 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -8,7 +8,8 @@ var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); -var TablesCacheEntry = require('../cache/model/database_tables_entry'); + +var QueryTables = require('cartodb-query-tables'); /** * @param {AuthApi} authApi @@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry'); * @param {WidgetBackend} widgetBackend * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi - * @param {QueryTablesApi} queryTablesApi * @param {LayergroupAffectedTables} layergroupAffectedTables * @constructor */ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, - widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) { + widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) { BaseController.call(this, authApi, pgConnection); + this.pgConnection = pgConnection; this.mapStore = mapStore; this.tileBackend = tileBackend; this.previewBackend = previewBackend; @@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev this.widgetBackend = widgetBackend; this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; - this.queryTablesApi = queryTablesApi; this.layergroupAffectedTables = layergroupAffectedTables; } @@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h global.logger.warn('ERROR generating cache channel: ' + err); } if (!!affectedTables) { - var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables); - res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); - self.surrogateKeysCache.tag(res, tablesCacheEntry); + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + self.surrogateKeysCache.tag(res, affectedTables); } self.send(req, res, body, status, headers); } @@ -366,17 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg throw new Error("this request doesn't need an X-Cache-Channel generated"); } - self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel + step( + function getConnection() { + self.pgConnection.getConnection(user, this); + }, + function getAffectedTables(err, connection) { + assert.ifError(err); + + QueryTables.getAffectedTablesFromQuery(connection, sql, this); + }, + this + ); }, - function buildCacheChannel(err, tableNames) { + function buildCacheChannel(err, tables) { assert.ifError(err); + self.layergroupAffectedTables.set(dbName, layergroupId, tables); - self.layergroupAffectedTables.set(dbName, layergroupId, tableNames); - - return tableNames; + return tables; }, - function finish(err, affectedTables) { - callback(err, affectedTables); - } + callback ); }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 035537d8..ff164e9c 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var step = require('step'); var windshaft = require('windshaft'); +var QueryTables = require('cartodb-query-tables'); var util = require('util'); var BaseController = require('./base'); @@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig; var Datasource = windshaft.model.Datasource; var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -var TablesCacheEntry = require('../cache/model/database_tables_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); @@ -26,7 +26,6 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter') * @param {TemplateMaps} templateMaps * @param {MapBackend} mapBackend * @param metadataBackend - * @param {QueryTablesApi} queryTablesApi * @param {OverviewsMetadataApi} overviewsMetadataApi * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi @@ -34,7 +33,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter') * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, - queryTablesApi, overviewsMetadataApi, + overviewsMetadataApi, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) { BaseController.call(this, authApi, pgConnection); @@ -43,7 +42,6 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.templateMaps = templateMaps; this.mapBackend = mapBackend; this.metadataBackend = metadataBackend; - this.queryTablesApi = queryTablesApi; this.overviewsMetadataApi = overviewsMetadataApi; this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; @@ -216,7 +214,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn self.templateMaps, self.pgConnection, self.userLimitsApi, - self.queryTablesApi, self.namedLayersAdapter, cdbuser, req.params.template_id, @@ -318,43 +315,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la var layergroupId = layergroup.layergroupid; step( - function checkCachedAffectedTables() { - return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId); + function getPgConnection() { + self.pgConnection.getConnection(username, this); }, - function getAffectedTablesAndLastUpdatedTime(err, hasCache) { + function getAffectedTablesAndLastUpdatedTime(err, connection) { assert.ifError(err); - if (hasCache) { - var next = this; - var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId); - self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) { - if (err) { - return next(err); - } - return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime }); - }); - } else { - self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); - } + QueryTables.getAffectedTablesFromQuery(connection, sql, this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { if (req.profiler) { req.profiler.done('queryTablesAndLastUpdated'); } assert.ifError(err); - self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables); + // feed affected tables cache so it can be reused from, for instance, layergroup controller + self.layergroupAffectedTables.set(dbName, layergroupId, result); // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; - layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString(); + layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt(); + layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString(); + // TODO this should take into account several URL patterns + addWidgetsUrl(username, layergroup); if (req.method === 'GET') { - var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); var ttl = global.environment.varnish.layergroupTtl || 86400; res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); res.set('Last-Modified', (new Date()).toUTCString()); - res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel()); - if (result.affectedTables && result.affectedTables.length > 0) { - self.surrogateKeysCache.tag(res, tableCacheEntry); + res.set('X-Cache-Channel', result.getCacheChannel()); + if (result.tables && result.tables.length > 0) { + self.surrogateKeysCache.tag(res, result); } } diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 004282c9..38ae91e8 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -9,8 +9,6 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); -var TablesCacheEntry = require('../cache/model/database_tables_entry'); - function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { BaseController.call(this, authApi, pgConnection); @@ -44,7 +42,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header var self = this; - var dbName = req.params.dbname; step( function getAffectedTablesAndLastUpdatedTime() { namedMapProvider.getAffectedTablesAndLastUpdatedTime(this); @@ -54,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header if (err) { global.logger.log('ERROR generating cache channel: ' + err); } - if (!result || !!result.affectedTables) { + if (!result || !!result.tables) { // we increase cache control as we can invalidate it res.set('Cache-Control', 'public,max-age=31536000'); var lastModifiedDate; if (Number.isFinite(result.lastUpdatedTime)) { - lastModifiedDate = new Date(result.lastUpdatedTime); + lastModifiedDate = new Date(result.getLastUpdatedAt()); } else { lastModifiedDate = new Date(); } res.set('Last-Modified', lastModifiedDate.toUTCString()); - var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); - res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); - if (result.affectedTables.length > 0) { - self.surrogateKeysCache.tag(res, tablesCacheEntry); + res.set('X-Cache-Channel', result.getCacheChannel()); + if (result.tables.length > 0) { + self.surrogateKeysCache.tag(res, result); } } self.send(req, res, resource, 200); @@ -231,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap return next(null); } - var affectedTables = affectedTablesAndLastUpdate.affectedTables || []; + var affectedTables = affectedTablesAndLastUpdate.tables || []; if (affectedTables.length === 0) { return next(null); diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index 0c86ce29..3e09b466 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -5,17 +5,17 @@ var dot = require('dot'); var step = require('step'); var MapConfig = require('windshaft').model.MapConfig; var templateName = require('../../backends/template_maps').templateName; +var QueryTables = require('cartodb-query-tables'); /** * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter, +function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, namedLayersAdapter, owner, templateId, config, authToken, params) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.userLimitsApi = userLimitsApi; - this.queryTablesApi = queryTablesApi; this.namedLayersAdapter = namedLayersAdapter; this.owner = owner; @@ -256,7 +256,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi }, function getAffectedTables(err, sql) { assert.ifError(err); - self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this); + step( + function getConnection() { + self.pgConnection.getConnection(self.owner, this); + }, + function getAffectedTables(err, connection) { + assert.ifError(err); + QueryTables.getAffectedTablesFromQuery(connection, sql, this); + }, + this + ); }, function finish(err, result) { self.affectedTablesAndLastUpdate = result; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 9f9d9f69..39884850 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -19,7 +19,6 @@ var windshaft = require('windshaft'); var mapnik = windshaft.mapnik; var TemplateMaps = require('./backends/template_maps.js'); -var QueryTablesApi = require('./api/query_tables_api'); var OverviewsMetadataApi = require('./api/overviews_metadata_api'); var UserLimitsApi = require('./api/user_limits_api'); var AuthApi = require('./api/auth_api'); @@ -52,7 +51,6 @@ module.exports = function(serverOptions) { var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); - var queryTablesApi = new QueryTablesApi(pgQueryRunner); var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); var userLimitsApi = new UserLimitsApi(metadataBackend, { limits: { @@ -142,7 +140,7 @@ module.exports = function(serverOptions) { var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi); + var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi); ['update', 'delete'].forEach(function(eventType) { templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); }); @@ -166,7 +164,6 @@ module.exports = function(serverOptions) { new windshaft.backend.Widget(), surrogateKeysCache, userLimitsApi, - queryTablesApi, layergroupAffectedTablesCache ).register(app); @@ -176,7 +173,6 @@ module.exports = function(serverOptions) { templateMaps, mapBackend, metadataBackend, - queryTablesApi, overviewsMetadataApi, surrogateKeysCache, userLimitsApi, diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 23e07d92..373ce17c 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -470,6 +470,11 @@ "from": "lzma@>=1.3.7 <1.4.0", "resolved": "https://registry.npmjs.org/lzma/-/lzma-1.3.7.tgz" }, + "cartodb-query-tables": { + "version": "0.1.0", + "from": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master", + "resolved": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" + }, "node-statsd": { "version": "0.0.7", "from": "node-statsd@>=0.0.7 <0.1.0", diff --git a/package.json b/package.json index b35b6099..60eb890e 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "redis-mpool": "~0.4.0", "lru-cache": "2.6.5", "lzma": "~1.3.7", - "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb" + "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb", + "cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" }, "devDependencies": { "istanbul": "~0.3.6", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 5375fc87..2dbfbbbd 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry'); +var QueryTables = require('cartodb-query-tables'); ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) { @@ -274,9 +274,9 @@ describe(suiteName, function() { var parsedBody = JSON.parse(res.body); expected_token = parsedBody.layergroupid.split(':')[0]; helper.checkCache(res); - helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [ - 'public.test_table', - 'public.test_table_2' + helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([ + {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"}, + {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"}, ]).key().join(' ')); diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index 7728073b..e443f3aa 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -7,6 +7,7 @@ var _ = require('underscore'); var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); +var QueryTables = require('cartodb-query-tables'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); @@ -360,9 +361,11 @@ describe('tests from old api translated to multilayer', function() { keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; keysToDelete['user:localhost:mapviews:global'] = 5; - var runQueryFn = PgQueryRunner.prototype.run; - PgQueryRunner.prototype.run = function(username, query, callback) { - return callback(new Error('failed to query database for affected tables'), []); + var affectedFn = QueryTables.getAffectedTablesFromQuery; + QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) { + affectedFn({query: function(query, callback) { + return callback(new Error('fake error message'), []); + }}, username, query, callback); }; // reset internal cacheChannel cache @@ -387,7 +390,7 @@ describe('tests from old api translated to multilayer', function() { }, function(res) { assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); - PgQueryRunner.prototype.run = runQueryFn; + QueryTables.getAffectedTablesFromQuery = affectedFn; done(); } ); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 0e1bd644..ae997ca8 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -3,8 +3,8 @@ var _ = require('underscore'); var redis = require('redis'); var step = require('step'); var strftime = require('strftime'); +var QueryTables = require('cartodb-query-tables'); var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry'); -var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry'); var redis_stats_db = 5; // Pollute the PG environment to make sure @@ -1405,7 +1405,8 @@ describe('template_api', function() { // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); var expectedSurrogateKey = [ - new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(), + new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public', + table_name: 'test_table_private_1'}]).key(), new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key() ].join(' '); helper.checkSurrogateKey(res, expectedSurrogateKey); @@ -1488,7 +1489,8 @@ describe('template_api', function() { // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); var expectedSurrogateKey = [ - new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(), + new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public', + table_name: 'test_table_private_1'}]).key(), new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key() ].join(' '); helper.checkSurrogateKey(res, expectedSurrogateKey); diff --git a/test/integration/overviews-metadata-api.js b/test/integration/overviews-metadata-api.js index 6c054d1e..f1182a6c 100644 --- a/test/integration/overviews-metadata-api.js +++ b/test/integration/overviews-metadata-api.js @@ -7,20 +7,18 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api'); var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); describe('OverviewsMetadataApi', function() { - var queryTablesApi, overviewsMetadataApi; + var overviewsMetadataApi; before(function() { var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); - queryTablesApi = new QueryTablesApi(pgQueryRunner); overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); }); diff --git a/test/integration/query-tables-api.js b/test/integration/query-tables-api.js deleted file mode 100644 index d05b1ed0..00000000 --- a/test/integration/query-tables-api.js +++ /dev/null @@ -1,55 +0,0 @@ -require('../support/test_helper'); - -var assert = require('assert'); - -var RedisPool = require('redis-mpool'); -var cartodbRedis = require('cartodb-redis'); - -var PgConnection = require('../../lib/cartodb/backends/pg_connection'); -var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api'); - - -describe('QueryTablesApi', function() { - - var queryTablesApi; - - before(function() { - var redisPool = new RedisPool(global.environment.redis); - var metadataBackend = cartodbRedis({pool: redisPool}); - var pgConnection = new PgConnection(metadataBackend); - var pgQueryRunner = new PgQueryRunner(pgConnection); - queryTablesApi = new QueryTablesApi(pgQueryRunner); - }); - - // Check test/support/sql/windshaft.test.sql to understand where the values come from. - - it('should return an object with affected tables array and last updated time', function(done) { - var query = 'select * from test_table'; - queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) { - assert.ok(!err, err); - - assert.deepEqual(result, { - affectedTables: [ 'public.test_table' ], - lastUpdatedTime: 1234567890123 - }); - - done(); - }); - }); - - it('should work with private tables', function(done) { - var query = 'select * from test_table_private_1'; - queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) { - assert.ok(!err, err); - - assert.deepEqual(result, { - affectedTables: [ 'public.test_table_private_1' ], - lastUpdatedTime: 1234567890123 - }); - - done(); - }); - }); - -}); diff --git a/test/integration/query-tables.js b/test/integration/query-tables.js new file mode 100644 index 00000000..9cd76000 --- /dev/null +++ b/test/integration/query-tables.js @@ -0,0 +1,71 @@ +require('../support/test_helper'); + +var assert = require('assert'); + +var RedisPool = require('redis-mpool'); +var cartodbRedis = require('cartodb-redis'); + +var PgConnection = require('../../lib/cartodb/backends/pg_connection'); + +var QueryTables = require('cartodb-query-tables'); + + +describe('QueryTables', function() { + + var connection; + + before(function(done) { + var redisPool = new RedisPool(global.environment.redis); + var metadataBackend = cartodbRedis({pool: redisPool}); + var pgConnection = new PgConnection(metadataBackend); + pgConnection.getConnection('localhost', function(err, pgConnection) { + if (err) { + return done(err); + } + connection = pgConnection; + + return done(); + }); + }); + + // Check test/support/sql/windshaft.test.sql to understand where the values come from. + + it('should return an object with affected tables array and last updated time', function(done) { + var query = 'select * from test_table'; + QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) { + assert.ok(!err, err); + + assert.equal(result.getLastUpdatedAt(), 1234567890123); + + assert.equal(result.tables.length, 1); + assert.deepEqual(result.tables[0], { + dbname: 'test_windshaft_cartodb_user_1_db', + schema_name: 'public', + table_name: 'test_table', + updated_at: new Date(1234567890123) + }); + + done(); + }); + }); + + it('should work with private tables', function(done) { + var query = 'select * from test_table_private_1'; + QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) { + assert.ok(!err, err); + + assert.equal(result.getLastUpdatedAt(), 1234567890123); + + assert.equal(result.tables.length, 1); + assert.deepEqual(result.tables[0], { + dbname: 'test_windshaft_cartodb_user_1_db', + schema_name: 'public', + table_name: 'test_table_private_1', + updated_at: new Date(1234567890123) + }); + + done(); + }); + }); + +}); \ No newline at end of file diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 26e4c0b9..f573e819 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -78,9 +78,15 @@ if test x"$PREPARE_PGSQL" = xyes; then sed "s/:TESTPASS/${TESTPASS}/" | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql - cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql sql/CDB_Overviews.sql | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + + SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews' + for i in ${SQL_SCRIPTS} + do + curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql + cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \ + | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + done fi diff --git a/test/support/sql/CDB_Overviews.sql b/test/support/sql/CDB_Overviews.sql deleted file mode 100644 index 654bd9d2..00000000 --- a/test/support/sql/CDB_Overviews.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Mockup for CDB_Overviews -CREATE OR REPLACE FUNCTION CDB_Overviews(table_names regclass[]) -RETURNS TABLE(base_table regclass, z integer, overview_table regclass) -AS $$ - BEGIN - IF (SELECT 'test_table_overviews'::regclass = ANY (table_names)) THEN - RETURN QUERY - SELECT 'test_table_overviews'::regclass AS base_table, 1 AS z, '_vovw_1_test_table_overviews'::regclass AS overview_table - UNION ALL - SELECT 'test_table_overviews'::regclass AS base_table, 2 AS z, '_vovw_2_test_table_overviews'::regclass AS overview_table; - ELSE - RETURN; - END IF; - END -$$ LANGUAGE PLPGSQL; - -CREATE OR REPLACE FUNCTION CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$ -BEGIN - CASE - WHEN scaleDenominator > 500000000 THEN RETURN 0; - WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1; - WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2; - WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3; - WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4; - WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5; - WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6; - WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7; - WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8; - WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9; - WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10; - WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11; - WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12; - WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13; - WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14; - WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15; - WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16; - WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17; - WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18; - WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19; - WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20; - WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21; - WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22; - WHEN scaleDenominator <= 100 THEN RETURN 23; - END CASE; -END -$$ LANGUAGE plpgsql IMMUTABLE; diff --git a/test/support/sql/CDB_QueryTables.sql b/test/support/sql/CDB_QueryTables.sql deleted file mode 100644 index c7cfa64b..00000000 --- a/test/support/sql/CDB_QueryTables.sql +++ /dev/null @@ -1,78 +0,0 @@ --- Return an array of table names scanned by a given query --- --- Requires PostgreSQL 9.x+ --- -CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text) -RETURNS text[] -AS $$ -DECLARE - exp XML; - tables text[]; - rec RECORD; - rec2 RECORD; -BEGIN - - tables := '{}'; - - FOR rec IN SELECT CDB_QueryStatements(query) q LOOP - - IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN - --RAISE WARNING 'Skipping %', rec.q; - CONTINUE; - END IF; - - BEGIN - EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp; - EXCEPTION WHEN others THEN - -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as - -- the affected table ? - RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM; - RAISE EXCEPTION '%', SQLERRM; - CONTINUE; - END; - - -- Now need to extract all values of - - -- RAISE DEBUG 'Explain: %', exp; - - FOR rec2 IN WITH - inp AS ( - SELECT - xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x, - xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s - ) - SELECT unnest(x)::text as p, unnest(s)::text as sc from inp - LOOP - -- RAISE DEBUG 'tab: %', rec2.p; - -- RAISE DEBUG 'sc: %', rec2.sc; - tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p))); - END LOOP; - - -- RAISE DEBUG 'Tables: %', tables; - - END LOOP; - - -- RAISE DEBUG 'Tables: %', tables; - - -- Remove duplicates and sort by name - IF array_upper(tables, 1) > 0 THEN - WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p ) - SELECT array_agg(p) from dist into tables; - END IF; - - --RAISE DEBUG 'Tables: %', tables; - - return tables; -END -$$ LANGUAGE 'plpgsql' VOLATILE STRICT; - - --- Keep CDB_QueryTables with same signature for backwards compatibility. --- It should probably be removed in the future. -CREATE OR REPLACE FUNCTION CDB_QueryTables(query text) -RETURNS name[] -AS $$ -BEGIN - RETURN CDB_QueryTablesText(query)::name[]; -END -$$ LANGUAGE 'plpgsql' VOLATILE STRICT; diff --git a/test/support/sql/CDB_QueryStatements.sql b/test/support/sql/_CDB_QueryStatements.sql similarity index 100% rename from test/support/sql/CDB_QueryStatements.sql rename to test/support/sql/_CDB_QueryStatements.sql diff --git a/test/support/test_helper.js b/test/support/test_helper.js index e1e64cbc..de9a6e3c 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -63,7 +63,15 @@ function checkCache(res) { function checkSurrogateKey(res, expectedKey) { assert.ok(res.headers.hasOwnProperty('surrogate-key')); - assert.equal(res.headers['surrogate-key'], expectedKey); + + function createSet(keys, key) { + keys[key] = true; + return keys; + } + var keys = res.headers['surrogate-key'].split(' ').reduce(createSet, {}); + var expectedKeys = expectedKey.split(' ').reduce(createSet, {}); + + assert.deepEqual(keys, expectedKeys); } var redisClient;