Use node-cartodb-query-tables library

lib/cartodb/api/query_tables_api.js

@@ -53,32 +53,6 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
     });
 };
 
-QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
-    if (!Array.isArray(tableNames) || tableNames.length === 0) {
-        return callback(null, 0);
-    }
-
-    var query = [
-        'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
-            tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
-        '])'
-    ].join(' ');
-
-    this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
-        if (err) {
-            var msg = err.message ? err.message : err;
-            return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
-        }
-        // when the table has not updated_at means it hasn't been changed so a default last_updated is set
-        var lastUpdated = 0;
-        if (rows.length !== 0) {
-            lastUpdated = rows[0].max || 0;
-        }
-
-        return callback(null, lastUpdated*1000);
-    });
-};
 
 function prepareSql(sql) {
     return sql

lib/cartodb/backends/pg_connection.js

@@ -1,5 +1,6 @@
 var assert = require('assert');
 var step = require('step');
+var PSQL = require('cartodb-psql');
 var _ = require('underscore');
 
 function PgConnection(metadataBackend) {
@@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
         }
     );
 };
+
+
+/**
+ * Returns a `cartodb-psql` object for a given username.
+ * @param {String} username
+ * @param {Function} callback function({Error}, {PSQL})
+ */
+
+PgConnection.prototype.getConnection = function(username, callback) {
+    var self = this;
+
+    var params = {};
+
+    require('debug')('cachechan')("getConn1");
+    step(
+        function setAuth() {
+            self.setDBAuth(username, params, this);
+        },
+        function setConn(err) {
+            assert.ifError(err);
+            self.setDBConn(username, params, this);
+        },
+        function openConnection(err) {
+            assert.ifError(err);
+            return callback(err, new PSQL({
+                user: params.dbuser,
+                pass: params.dbpass,
+                host: params.dbhost,
+                port: params.dbport,
+                dbname: params.dbname
+            }));
+        }
+    );
+};

lib/cartodb/cache/model/database_tables_entry.js

@@ -1,26 +0,0 @@
-var crypto = require('crypto');
-
-function DatabaseTables(tables) {
-    this.namespace = 't';
-    this.tables = tables;
-}
-
-module.exports = DatabaseTables;
-
-
-DatabaseTables.prototype.key = function() {
-    return this.tables.map(function(table) {
-        return this.namespace + ':' + shortHashKey(table.dbname + ':' + table.table_name + '.' + table.schema_name);
-    }.bind(this)).sort();
-};
-
-DatabaseTables.prototype.getCacheChannel = function() {
-    var key = this.tables.map(function(table) {
-        return table.dbname + ':' + table.schema_name + "." + table.table_name;
-    }).sort().join(";;");
-    return key;
-};
-
-function shortHashKey(target) {
-    return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
-}

lib/cartodb/controllers/layergroup.js

@@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
 var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
+
+var QueryTables = require('node-cartodb-query-tables');
 
 /**
  * @param {AuthApi} authApi
@@ -28,6 +29,7 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
                           widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
     BaseController.call(this, authApi, pgConnection);
 
+    this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
@@ -320,9 +322,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
                 global.logger.warn('ERROR generating cache channel: ' + err);
             }
             if (!!affectedTables) {
-                var tablesCacheEntry = new TablesCacheEntry(affectedTables);
+                res.set('X-Cache-Channel', affectedTables.getCacheChannel());
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
+                self.surrogateKeysCache.tag(res, affectedTables);
-                self.surrogateKeysCache.tag(res, tablesCacheEntry);
             }
             self.send(req, res, body, status, headers);
         }
@@ -366,11 +367,21 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
                 throw new Error("this request doesn't need an X-Cache-Channel generated");
             }
 
-            self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(user, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
         },
         function buildCacheChannel(err, tables) {
             assert.ifError(err);
-            self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables);
+            self.layergroupAffectedTables.set(dbName, layergroupId, tables);
 
             return tables;
         },
@@ -378,7 +389,7 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
             if(tables === undefined){
                 callback(err);
             }else{
-                callback(err, tables.affectedTables);
+                callback(err, tables);
             }
         }
     );

lib/cartodb/controllers/map.js

@@ -2,6 +2,7 @@ var _ = require('underscore');
 var assert = require('assert');
 var step = require('step');
 var windshaft = require('windshaft');
+var QueryTables = require('node-cartodb-query-tables');
 
 var util = require('util');
 var BaseController = require('./base');
@@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
 var Datasource = windshaft.model.Datasource;
 
 var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
 
 var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
 var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
@@ -329,31 +329,39 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
             //         return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
             //     });
             // } else {
-                self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
+                // self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
             //}
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(username, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
             if (req.profiler) {
                 req.profiler.done('queryTablesAndLastUpdated');
             }
             assert.ifError(err);
-            self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
+            self.layergroupAffectedTables.set(dbName, layergroupId, result);
 
             // last update for layergroup cache buster
-            layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
+            layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
-            layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
+            layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
 
             // TODO this should take into account several URL patterns
             addWidgetsUrl(username, layergroup);
-
             if (req.method === 'GET') {
-                var tableCacheEntry = new TablesCacheEntry(result.affectedTables);
                 var ttl = global.environment.varnish.layergroupTtl || 86400;
                 res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
                 res.set('Last-Modified', (new Date()).toUTCString());
-                res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
+                res.set('X-Cache-Channel', result.getCacheChannel());
-                if (result.affectedTables && result.affectedTables.length > 0) {
+                if (result.tables && result.tables.length > 0) {
-                    self.surrogateKeysCache.tag(res, tableCacheEntry);
+                    self.surrogateKeysCache.tag(res, result);
                 }
             }
 

lib/cartodb/controllers/named_maps.js

@@ -9,8 +9,6 @@ var BaseController = require('./base');
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
-
 function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
     BaseController.call(this, authApi, pgConnection);
@@ -53,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
             if (err) {
                 global.logger.log('ERROR generating cache channel: ' + err);
             }
-            if (!result || !!result.affectedTables) {
+            if (!result || !!result.tables) {
                 // we increase cache control as we can invalidate it
                 res.set('Cache-Control', 'public,max-age=31536000');
 
                 var lastModifiedDate;
                 if (Number.isFinite(result.lastUpdatedTime)) {
-                    lastModifiedDate = new Date(result.lastUpdatedTime);
+                    lastModifiedDate = new Date(result.getLastUpdatedAt());
                 } else {
                     lastModifiedDate = new Date();
                 }
                 res.set('Last-Modified', lastModifiedDate.toUTCString());
 
-                var tablesCacheEntry = new TablesCacheEntry(result.affectedTables);
+                res.set('X-Cache-Channel', result.getCacheChannel());
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
+                if (result.tables.length > 0) {
-                if (result.affectedTables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
-                    self.surrogateKeysCache.tag(res, tablesCacheEntry);
                 }
             }
             self.send(req, res, resource, 200);
@@ -230,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
                     return next(null);
                 }
 
-                var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
+                var affectedTables = affectedTablesAndLastUpdate.tables || [];
 
                 if (affectedTables.length === 0) {
                     return next(null);

package.json

@@ -37,6 +37,7 @@
         "lru-cache": "2.6.5",
         "lzma": "~1.3.7",
         "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
+        "node-cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
     },
     "devDependencies": {
         "istanbul": "~0.3.6",

test/acceptance/multilayer.js

@@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
+var QueryTables = require('node-cartodb-query-tables');
 
 ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
 
@@ -262,7 +262,7 @@ describe(suiteName, function() {
           var parsedBody = JSON.parse(res.body);
           expected_token = parsedBody.layergroupid.split(':')[0];
           helper.checkCache(res);
-          helper.checkSurrogateKey(res, new TablesCacheEntry([
+          helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
             {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
             {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
           ]).key().join(' '));

test/acceptance/multilayer_server.js

@@ -7,6 +7,7 @@ var _ = require('underscore');
 var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
 
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var QueryTables = require('node-cartodb-query-tables');
 var CartodbWindshaft = require('../../lib/cartodb/server');
 var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
@@ -309,6 +310,7 @@ describe('tests from old api translated to multilayer', function() {
 
     it("creates layergroup fails when postgresql queries fail to figure affected tables in query",  function(done) {
 
+
         var runQueryFn = PgQueryRunner.prototype.run;
         PgQueryRunner.prototype.run = function(username, query, callback) {
             return callback(new Error('fake error message'), []);
@@ -343,6 +345,7 @@ describe('tests from old api translated to multilayer', function() {
     });
 
     it("tile requests works when postgresql queries fail to figure affected tables in query",  function(done) {
+
         var layergroup =  singleLayergroupConfig('select * from gadm4', '#gadm4 { marker-fill: red; }');
         assert.response(server,
             {
@@ -360,9 +363,11 @@ describe('tests from old api translated to multilayer', function() {
                 keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
                 keysToDelete['user:localhost:mapviews:global'] = 5;
 
-                var runQueryFn = PgQueryRunner.prototype.run;
+                var affectedFn = QueryTables.getAffectedTablesFromQuery;
-                PgQueryRunner.prototype.run = function(username, query, callback) {
+                QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
-                    return callback(new Error('failed to query database for affected tables'), []);
+                    affectedFn({query: function(query, callback) {
+                        return callback(new Error('fake error message'), []);
+                    }}, username, query, callback);
                 };
 
                 // reset internal cacheChannel cache
@@ -387,7 +392,7 @@ describe('tests from old api translated to multilayer', function() {
                     },
                     function(res) {
                         assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
-                        PgQueryRunner.prototype.run = runQueryFn;
+                        QueryTables.getAffectedTablesFromQuery = affectedFn;
                         done();
                     }
                 );

test/acceptance/templates.js

@@ -3,8 +3,8 @@ var _           = require('underscore');
 var redis       = require('redis');
 var step        = require('step');
 var strftime    = require('strftime');
+var QueryTables = require('node-cartodb-query-tables');
 var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
 var redis_stats_db = 5;
 
 // Pollute the PG environment to make sure
@@ -1393,7 +1393,7 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-            new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+            new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
                                    table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');
@@ -1477,7 +1477,7 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-            new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+            new QueryTables.DatabaseTableEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
                                    table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');