Merge pull request #388 from CartoDB/new_querytables_library

Use new querytables library

.travis.yml

@@ -12,6 +12,7 @@ addons:
 before_install:
   - npm install -g npm@2
   - createdb template_postgis
+  - createuser publicuser
   - psql -c "CREATE EXTENSION postgis" template_postgis
 
 env:

lib/cartodb/api/query_tables_api.js

@@ -1,96 +0,0 @@
-function QueryTablesApi(pgQueryRunner) {
-    this.pgQueryRunner = pgQueryRunner;
-}
-
-var affectedTableRegexCache = {
-    bbox: /!bbox!/g,
-    scale_denominator: /!scale_denominator!/g,
-    pixel_width: /!pixel_width!/g,
-    pixel_height: /!pixel_height!/g
-};
-
-module.exports = QueryTablesApi;
-
-
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
-    var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
-
-    this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) {
-        if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch source tables: ' + msg));
-            return;
-        }
-
-        // This is an Array, so no need to split into parts
-        var tableNames = rows[0].cdb_querytablestext;
-        return callback(null, tableNames);
-    });
-};
-
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
-    var query = [
-        'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
-        ')',
-        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m',
-        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
-    ].join(' ');
-
-    this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
-        if (err || rows.length === 0) {
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch affected tables or last updated time: ' + msg));
-            return;
-        }
-
-        var result = rows[0];
-
-        // This is an Array, so no need to split into parts
-        var tableNames = result.tablenames;
-
-        var lastUpdatedTime = result.max || 0;
-
-        callback(null, {
-            affectedTables: tableNames,
-            lastUpdatedTime: lastUpdatedTime * 1000
-        });
-    });
-};
-
-QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
-    if (!Array.isArray(tableNames) || tableNames.length === 0) {
-        return callback(null, 0);
-    }
-
-    var query = [
-        'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
-            tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
-        '])'
-    ].join(' ');
-
-    this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
-        if (err) {
-            var msg = err.message ? err.message : err;
-            return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
-        }
-        // when the table has not updated_at means it hasn't been changed so a default last_updated is set
-        var lastUpdated = 0;
-        if (rows.length !== 0) {
-            lastUpdated = rows[0].max || 0;
-        }
-
-        return callback(null, lastUpdated*1000);
-    });
-};
-
-function prepareSql(sql) {
-    return sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.scale_denominator, '0')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
-}

lib/cartodb/api/tables_extent_api.js

@@ -13,13 +13,9 @@ module.exports = TablesExtentApi;
  *  `table_name` format as valid input
  * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
  */
-TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) {
+TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
-    var estimatedExtentSQLs = tableNames.map(function(tableName) {
+    var estimatedExtentSQLs = tables.map(function(table) {
-        var schemaTable = tableName.split('.');
+        return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
-        if (schemaTable.length > 1) {
-            return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')";
-        }
-        return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')";
     });
 
     var query = [

lib/cartodb/backends/pg_connection.js

@@ -1,5 +1,6 @@
 var assert = require('assert');
 var step = require('step');
+var PSQL = require('cartodb-psql');
 var _ = require('underscore');
 
 function PgConnection(metadataBackend) {
@@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
         }
     );
 };
+
+
+/**
+ * Returns a `cartodb-psql` object for a given username.
+ * @param {String} username
+ * @param {Function} callback function({Error}, {PSQL})
+ */
+
+PgConnection.prototype.getConnection = function(username, callback) {
+    var self = this;
+
+    var params = {};
+
+    require('debug')('cachechan')("getConn1");
+    step(
+        function setAuth() {
+            self.setDBAuth(username, params, this);
+        },
+        function setConn(err) {
+            assert.ifError(err);
+            self.setDBConn(username, params, this);
+        },
+        function openConnection(err) {
+            assert.ifError(err);
+            return callback(err, new PSQL({
+                user: params.dbuser,
+                pass: params.dbpass,
+                host: params.dbhost,
+                port: params.dbport,
+                dbname: params.dbname
+            }));
+        }
+    );
+};

lib/cartodb/cache/model/database_tables_entry.js

@@ -1,24 +0,0 @@
-var crypto = require('crypto');
-
-function DatabaseTables(dbName, tableNames) {
-    this.namespace = 't';
-    this.dbName = dbName;
-    this.tableNames = tableNames;
-}
-
-module.exports = DatabaseTables;
-
-
-DatabaseTables.prototype.key = function() {
-    return this.tableNames.map(function(tableName) {
-        return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
-    }.bind(this));
-};
-
-DatabaseTables.prototype.getCacheChannel = function() {
-    return this.dbName + ':' + this.tableNames.join(',');
-};
-
-function shortHashKey(target) {
-    return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
-}

lib/cartodb/cache/named_map_provider_cache.js

@@ -7,11 +7,10 @@ var queue = require('queue-async');
 
 var LruCache = require("lru-cache");
 
-function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
+function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
 
@@ -30,7 +29,6 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
             this.templateMaps,
             this.pgConnection,
             this.userLimitsApi,
-            this.queryTablesApi,
             this.namedLayersAdapter,
             user,
             templateId,

lib/cartodb/controllers/layergroup.js

@@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
 var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
+
+var QueryTables = require('cartodb-query-tables');
 
 /**
  * @param {AuthApi} authApi
@@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
  * @param {WidgetBackend} widgetBackend
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
- * @param {QueryTablesApi} queryTablesApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @constructor
  */
 function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
-                          widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
+                          widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
     BaseController.call(this, authApi, pgConnection);
 
+    this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
@@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
     this.widgetBackend = widgetBackend;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
 }
 
@@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
                 global.logger.warn('ERROR generating cache channel: ' + err);
             }
             if (!!affectedTables) {
-                var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
+                res.set('X-Cache-Channel', affectedTables.getCacheChannel());
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
+                self.surrogateKeysCache.tag(res, affectedTables);
-                self.surrogateKeysCache.tag(res, tablesCacheEntry);
             }
             self.send(req, res, body, status, headers);
         }
@@ -366,17 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
                 throw new Error("this request doesn't need an X-Cache-Channel generated");
             }
 
-            self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(user, this);
                 },
-        function buildCacheChannel(err, tableNames) {
+                function getAffectedTables(err, connection) {
                     assert.ifError(err);
 
-            self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
-
-            return tableNames;
                 },
-        function finish(err, affectedTables) {
+                this
-            callback(err, affectedTables);
+            );
-        }
+        },
+        function buildCacheChannel(err, tables) {
+            assert.ifError(err);
+            self.layergroupAffectedTables.set(dbName, layergroupId, tables);
+
+            return tables;
+        },
+        callback
     );
 };

lib/cartodb/controllers/map.js

@@ -2,6 +2,7 @@ var _ = require('underscore');
 var assert = require('assert');
 var step = require('step');
 var windshaft = require('windshaft');
+var QueryTables = require('cartodb-query-tables');
 
 var util = require('util');
 var BaseController = require('./base');
@@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
 var Datasource = windshaft.model.Datasource;
 
 var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
 
 var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
 var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
@@ -26,7 +26,6 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
  * @param {TemplateMaps} templateMaps
  * @param {MapBackend} mapBackend
  * @param metadataBackend
- * @param {QueryTablesApi} queryTablesApi
  * @param {OverviewsMetadataApi} overviewsMetadataApi
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
@@ -34,7 +33,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
  * @constructor
  */
 function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
-                       queryTablesApi, overviewsMetadataApi,
+                       overviewsMetadataApi,
                        surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
 
     BaseController.call(this, authApi, pgConnection);
@@ -43,7 +42,6 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.templateMaps = templateMaps;
     this.mapBackend = mapBackend;
     this.metadataBackend = metadataBackend;
-    this.queryTablesApi = queryTablesApi;
     this.overviewsMetadataApi = overviewsMetadataApi;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
@@ -216,7 +214,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 self.templateMaps,
                 self.pgConnection,
                 self.userLimitsApi,
-                self.queryTablesApi,
                 self.namedLayersAdapter,
                 cdbuser,
                 req.params.template_id,
@@ -318,43 +315,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
     var layergroupId = layergroup.layergroupid;
 
     step(
-        function checkCachedAffectedTables() {
+        function getPgConnection() {
-            return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
+            self.pgConnection.getConnection(username, this);
         },
-        function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
+        function getAffectedTablesAndLastUpdatedTime(err, connection) {
             assert.ifError(err);
-            if (hasCache) {
+            QueryTables.getAffectedTablesFromQuery(connection, sql, this);
-                var next = this;
-                var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
-                self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
-                    if (err) {
-                        return next(err);
-                    }
-                    return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
-                });
-            } else {
-                self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
-            }
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
             if (req.profiler) {
                 req.profiler.done('queryTablesAndLastUpdated');
             }
             assert.ifError(err);
-            self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
+            // feed affected tables cache so it can be reused from, for instance, layergroup controller
+            self.layergroupAffectedTables.set(dbName, layergroupId, result);
 
             // last update for layergroup cache buster
-            layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
+            layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
-            layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
+            layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
 
+            // TODO this should take into account several URL patterns
+            addWidgetsUrl(username, layergroup);
             if (req.method === 'GET') {
-                var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
                 var ttl = global.environment.varnish.layergroupTtl || 86400;
                 res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
                 res.set('Last-Modified', (new Date()).toUTCString());
-                res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
+                res.set('X-Cache-Channel', result.getCacheChannel());
-                if (result.affectedTables && result.affectedTables.length > 0) {
+                if (result.tables && result.tables.length > 0) {
-                    self.surrogateKeysCache.tag(res, tableCacheEntry);
+                    self.surrogateKeysCache.tag(res, result);
                 }
             }
 

lib/cartodb/controllers/named_maps.js

@@ -9,8 +9,6 @@ var BaseController = require('./base');
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
-
 function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
     BaseController.call(this, authApi, pgConnection);
@@ -44,7 +42,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
 
     var self = this;
 
-    var dbName = req.params.dbname;
     step(
         function getAffectedTablesAndLastUpdatedTime() {
             namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
@@ -54,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
             if (err) {
                 global.logger.log('ERROR generating cache channel: ' + err);
             }
-            if (!result || !!result.affectedTables) {
+            if (!result || !!result.tables) {
                 // we increase cache control as we can invalidate it
                 res.set('Cache-Control', 'public,max-age=31536000');
 
                 var lastModifiedDate;
                 if (Number.isFinite(result.lastUpdatedTime)) {
-                    lastModifiedDate = new Date(result.lastUpdatedTime);
+                    lastModifiedDate = new Date(result.getLastUpdatedAt());
                 } else {
                     lastModifiedDate = new Date();
                 }
                 res.set('Last-Modified', lastModifiedDate.toUTCString());
 
-                var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
+                res.set('X-Cache-Channel', result.getCacheChannel());
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
+                if (result.tables.length > 0) {
-                if (result.affectedTables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
-                    self.surrogateKeysCache.tag(res, tablesCacheEntry);
                 }
             }
             self.send(req, res, resource, 200);
@@ -231,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
                     return next(null);
                 }
 
-                var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
+                var affectedTables = affectedTablesAndLastUpdate.tables || [];
 
                 if (affectedTables.length === 0) {
                     return next(null);

lib/cartodb/models/mapconfig/named_map_provider.js

@@ -5,17 +5,17 @@ var dot = require('dot');
 var step = require('step');
 var MapConfig = require('windshaft').model.MapConfig;
 var templateName = require('../../backends/template_maps').templateName;
+var QueryTables = require('cartodb-query-tables');
 
 /**
  * @constructor
  * @type {NamedMapMapConfigProvider}
  */
-function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter,
+function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, namedLayersAdapter,
                                    owner, templateId, config, authToken, params) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
     this.namedLayersAdapter = namedLayersAdapter;
 
     this.owner = owner;
@@ -256,7 +256,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
         },
         function getAffectedTables(err, sql) {
             assert.ifError(err);
-            self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(self.owner, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
         },
         function finish(err, result) {
             self.affectedTablesAndLastUpdate = result;

lib/cartodb/server.js

@@ -19,7 +19,6 @@ var windshaft = require('windshaft');
 var mapnik = windshaft.mapnik;
 
 var TemplateMaps = require('./backends/template_maps.js');
-var QueryTablesApi = require('./api/query_tables_api');
 var OverviewsMetadataApi = require('./api/overviews_metadata_api');
 var UserLimitsApi = require('./api/user_limits_api');
 var AuthApi = require('./api/auth_api');
@@ -52,7 +51,6 @@ module.exports = function(serverOptions) {
     var metadataBackend = cartodbRedis({pool: redisPool});
     var pgConnection = new PgConnection(metadataBackend);
     var pgQueryRunner = new PgQueryRunner(pgConnection);
-    var queryTablesApi = new QueryTablesApi(pgQueryRunner);
     var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
     var userLimitsApi = new UserLimitsApi(metadataBackend, {
         limits: {
@@ -142,7 +140,7 @@ module.exports = function(serverOptions) {
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
-    var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
+    var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi);
     ['update', 'delete'].forEach(function(eventType) {
         templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
     });
@@ -166,7 +164,6 @@ module.exports = function(serverOptions) {
         new windshaft.backend.Widget(),
         surrogateKeysCache,
         userLimitsApi,
-        queryTablesApi,
         layergroupAffectedTablesCache
     ).register(app);
 
@@ -176,7 +173,6 @@ module.exports = function(serverOptions) {
         templateMaps,
         mapBackend,
         metadataBackend,
-        queryTablesApi,
         overviewsMetadataApi,
         surrogateKeysCache,
         userLimitsApi,

npm-shrinkwrap.json

@@ -470,6 +470,11 @@
       "from": "lzma@>=1.3.7 <1.4.0",
       "resolved": "https://registry.npmjs.org/lzma/-/lzma-1.3.7.tgz"
     },
+    "cartodb-query-tables": {
+      "version": "0.1.0",
+      "from": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master",
+      "resolved": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
+    },
     "node-statsd": {
       "version": "0.0.7",
       "from": "node-statsd@>=0.0.7 <0.1.0",

package.json

@@ -36,7 +36,8 @@
         "redis-mpool": "~0.4.0",
         "lru-cache": "2.6.5",
         "lzma": "~1.3.7",
-        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
+        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
+        "cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
     },
     "devDependencies": {
         "istanbul": "~0.3.6",

test/acceptance/multilayer.js

@@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
+var QueryTables = require('cartodb-query-tables');
 
 ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
 
@@ -274,9 +274,9 @@ describe(suiteName, function() {
           var parsedBody = JSON.parse(res.body);
           expected_token = parsedBody.layergroupid.split(':')[0];
           helper.checkCache(res);
-          helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
+          helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
-              'public.test_table',
+            {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
-              'public.test_table_2'
+            {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
           ]).key().join(' '));
 
 

test/acceptance/multilayer_server.js

@@ -7,6 +7,7 @@ var _ = require('underscore');
 var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
 
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var QueryTables = require('cartodb-query-tables');
 var CartodbWindshaft = require('../../lib/cartodb/server');
 var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
@@ -360,9 +361,11 @@ describe('tests from old api translated to multilayer', function() {
                 keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
                 keysToDelete['user:localhost:mapviews:global'] = 5;
 
-                var runQueryFn = PgQueryRunner.prototype.run;
+                var affectedFn = QueryTables.getAffectedTablesFromQuery;
-                PgQueryRunner.prototype.run = function(username, query, callback) {
+                QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
-                    return callback(new Error('failed to query database for affected tables'), []);
+                    affectedFn({query: function(query, callback) {
+                        return callback(new Error('fake error message'), []);
+                    }}, username, query, callback);
                 };
 
                 // reset internal cacheChannel cache
@@ -387,7 +390,7 @@ describe('tests from old api translated to multilayer', function() {
                     },
                     function(res) {
                         assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
-                        PgQueryRunner.prototype.run = runQueryFn;
+                        QueryTables.getAffectedTablesFromQuery = affectedFn;
                         done();
                     }
                 );

test/acceptance/templates.js

@@ -3,8 +3,8 @@ var _           = require('underscore');
 var redis       = require('redis');
 var step        = require('step');
 var strftime    = require('strftime');
+var QueryTables = require('cartodb-query-tables');
 var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
 var redis_stats_db = 5;
 
 // Pollute the PG environment to make sure
@@ -1405,7 +1405,8 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-              new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
+            new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+                                   table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');
           helper.checkSurrogateKey(res, expectedSurrogateKey);
@@ -1488,7 +1489,8 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-              new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
+            new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+                                   table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');
           helper.checkSurrogateKey(res, expectedSurrogateKey);

test/integration/overviews-metadata-api.js

@@ -7,20 +7,18 @@ var cartodbRedis = require('cartodb-redis');
 
 var PgConnection = require('../../lib/cartodb/backends/pg_connection');
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
-var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
 var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
 
 
 describe('OverviewsMetadataApi', function() {
 
-    var queryTablesApi, overviewsMetadataApi;
+    var overviewsMetadataApi;
 
     before(function() {
         var redisPool = new RedisPool(global.environment.redis);
         var metadataBackend = cartodbRedis({pool: redisPool});
         var pgConnection = new PgConnection(metadataBackend);
         var pgQueryRunner = new PgQueryRunner(pgConnection);
-        queryTablesApi = new QueryTablesApi(pgQueryRunner);
         overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
     });
 

test/integration/query-tables-api.js

@@ -1,55 +0,0 @@
-require('../support/test_helper');
-
-var assert = require('assert');
-
-var RedisPool = require('redis-mpool');
-var cartodbRedis = require('cartodb-redis');
-
-var PgConnection = require('../../lib/cartodb/backends/pg_connection');
-var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
-var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
-
-
-describe('QueryTablesApi', function() {
-
-    var queryTablesApi;
-
-    before(function() {
-        var redisPool = new RedisPool(global.environment.redis);
-        var metadataBackend = cartodbRedis({pool: redisPool});
-        var pgConnection = new PgConnection(metadataBackend);
-        var pgQueryRunner = new PgQueryRunner(pgConnection);
-        queryTablesApi = new QueryTablesApi(pgQueryRunner);
-    });
-
-    // Check test/support/sql/windshaft.test.sql to understand where the values come from.
-
-    it('should return an object with affected tables array and last updated time', function(done) {
-        var query = 'select * from test_table';
-        queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
-            assert.ok(!err, err);
-
-            assert.deepEqual(result, {
-                affectedTables: [ 'public.test_table' ],
-                lastUpdatedTime: 1234567890123
-            });
-
-            done();
-        });
-    });
-
-    it('should work with private tables', function(done) {
-        var query = 'select * from test_table_private_1';
-        queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
-            assert.ok(!err, err);
-
-            assert.deepEqual(result, {
-                affectedTables: [ 'public.test_table_private_1' ],
-                lastUpdatedTime: 1234567890123
-            });
-
-            done();
-        });
-    });
-
-});

test/integration/query-tables.js

@@ -0,0 +1,71 @@
+require('../support/test_helper');
+
+var assert = require('assert');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var PgConnection = require('../../lib/cartodb/backends/pg_connection');
+
+var QueryTables = require('cartodb-query-tables');
+
+
+describe('QueryTables', function() {
+
+    var connection;
+
+    before(function(done) {
+        var redisPool = new RedisPool(global.environment.redis);
+        var metadataBackend = cartodbRedis({pool: redisPool});
+        var pgConnection = new PgConnection(metadataBackend);
+        pgConnection.getConnection('localhost', function(err, pgConnection) {
+            if (err) {
+                return done(err);
+            }
+            connection = pgConnection;
+
+            return done();
+        });
+    });
+
+    // Check test/support/sql/windshaft.test.sql to understand where the values come from.
+
+    it('should return an object with affected tables array and last updated time', function(done) {
+        var query = 'select * from test_table';
+        QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.equal(result.getLastUpdatedAt(), 1234567890123);
+
+            assert.equal(result.tables.length, 1);
+            assert.deepEqual(result.tables[0], {
+                dbname: 'test_windshaft_cartodb_user_1_db',
+                schema_name: 'public',
+                table_name: 'test_table',
+                updated_at: new Date(1234567890123)
+            });
+
+            done();
+        });
+    });
+
+    it('should work with private tables', function(done) {
+        var query = 'select * from test_table_private_1';
+        QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.equal(result.getLastUpdatedAt(), 1234567890123);
+
+            assert.equal(result.tables.length, 1);
+            assert.deepEqual(result.tables[0], {
+                dbname: 'test_windshaft_cartodb_user_1_db',
+                schema_name: 'public',
+                table_name: 'test_table_private_1',
+                updated_at: new Date(1234567890123)
+            });
+
+            done();
+        });
+    });
+
+});

test/support/prepare_db.sh

@@ -78,9 +78,15 @@ if test x"$PREPARE_PGSQL" = xyes; then
     sed "s/:TESTPASS/${TESTPASS}/" |
     psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
-  curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
+  cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
-  cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql sql/CDB_Overviews.sql |
+
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews'
+  for i in ${SQL_SCRIPTS}
+  do
+    curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
+    cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
+        | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  done
 
 fi
 

test/support/sql/CDB_Overviews.sql

@@ -1,46 +0,0 @@
--- Mockup for CDB_Overviews
-CREATE OR REPLACE FUNCTION CDB_Overviews(table_names regclass[])
-RETURNS TABLE(base_table regclass, z integer, overview_table regclass)
-AS $$
-  BEGIN
-    IF (SELECT 'test_table_overviews'::regclass = ANY (table_names)) THEN
-      RETURN QUERY
-        SELECT 'test_table_overviews'::regclass AS base_table, 1 AS z, '_vovw_1_test_table_overviews'::regclass AS overview_table
-        UNION ALL
-        SELECT 'test_table_overviews'::regclass AS base_table, 2 AS z, '_vovw_2_test_table_overviews'::regclass AS overview_table;
-    ELSE
-      RETURN;
-    END IF;
-  END
-$$ LANGUAGE PLPGSQL;
-
-CREATE OR REPLACE FUNCTION CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
-BEGIN
-  CASE
-    WHEN scaleDenominator > 500000000 THEN RETURN 0;
-    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
-    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
-    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
-    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
-    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
-    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
-    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
-    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
-    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
-    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
-    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
-    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
-    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
-    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
-    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
-    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
-    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
-    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
-    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
-    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
-    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
-    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
-    WHEN scaleDenominator <= 100 THEN RETURN 23;
-  END CASE;
-END
-$$ LANGUAGE plpgsql IMMUTABLE;

test/support/sql/CDB_QueryTables.sql

@@ -1,78 +0,0 @@
--- Return an array of table names scanned by a given query
---
--- Requires PostgreSQL 9.x+
---
-CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
-RETURNS text[]
-AS $$
-DECLARE
-  exp XML;
-  tables text[];
-  rec RECORD;
-  rec2 RECORD;
-BEGIN
-  
-  tables := '{}';
-
-  FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
-
-    IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
-        --RAISE WARNING 'Skipping %', rec.q;
-        CONTINUE;
-    END IF;
-
-    BEGIN
-      EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
-    EXCEPTION WHEN others THEN
-      -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
-      --       the affected table ?
-      RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
-      RAISE EXCEPTION '%', SQLERRM;
-      CONTINUE;
-    END;
-
-    -- Now need to extract all values of <Relation-Name>
-
-    -- RAISE DEBUG 'Explain: %', exp;
-
-    FOR rec2 IN WITH
-      inp AS (
-        SELECT
-          xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
-          xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
-      )
-      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
-    LOOP
-      -- RAISE DEBUG 'tab: %', rec2.p;
-      -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
-    END LOOP;
-
-    -- RAISE DEBUG 'Tables: %', tables;
-
-  END LOOP;
-
-  -- RAISE DEBUG 'Tables: %', tables;
-
-  -- Remove duplicates and sort by name
-  IF array_upper(tables, 1) > 0 THEN
-    WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
-       SELECT array_agg(p) from dist into tables;
-  END IF;
-
-  --RAISE DEBUG 'Tables: %', tables;
-
-  return tables;
-END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
-
-
--- Keep CDB_QueryTables with same signature for backwards compatibility.
--- It should probably be removed in the future.
-CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
-RETURNS name[]
-AS $$
-BEGIN
-  RETURN CDB_QueryTablesText(query)::name[];
-END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

test/support/test_helper.js

@@ -63,7 +63,15 @@ function checkCache(res) {
 
 function checkSurrogateKey(res, expectedKey) {
     assert.ok(res.headers.hasOwnProperty('surrogate-key'));
-    assert.equal(res.headers['surrogate-key'], expectedKey);
+
+    function createSet(keys, key) {
+        keys[key] = true;
+        return keys;
+    }
+    var keys = res.headers['surrogate-key'].split(' ').reduce(createSet, {});
+    var expectedKeys = expectedKey.split(' ').reduce(createSet, {});
+
+    assert.deepEqual(keys, expectedKeys);
 }
 
 var redisClient;