From cf06ff86c2a7df53bf7afcbde89414525ef936b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Mon, 22 Feb 2016 11:40:25 +0100 Subject: [PATCH] Use node-cartodb-query-tables library --- lib/cartodb/api/query_tables_api.js | 26 -------------- lib/cartodb/backends/pg_connection.js | 35 +++++++++++++++++++ .../cache/model/database_tables_entry.js | 26 -------------- lib/cartodb/controllers/layergroup.js | 25 +++++++++---- lib/cartodb/controllers/map.js | 28 +++++++++------ lib/cartodb/controllers/named_maps.js | 15 ++++---- package.json | 1 + test/acceptance/multilayer.js | 4 +-- test/acceptance/multilayer_server.js | 13 ++++--- test/acceptance/templates.js | 6 ++-- 10 files changed, 92 insertions(+), 87 deletions(-) delete mode 100644 lib/cartodb/cache/model/database_tables_entry.js diff --git a/lib/cartodb/api/query_tables_api.js b/lib/cartodb/api/query_tables_api.js index a94d6411..bf85b1a0 100644 --- a/lib/cartodb/api/query_tables_api.js +++ b/lib/cartodb/api/query_tables_api.js @@ -53,32 +53,6 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam }); }; -QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) { - if (!Array.isArray(tableNames) || tableNames.length === 0) { - return callback(null, 0); - } - - var query = [ - 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max', - 'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[', - tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','), - '])' - ].join(' '); - - this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) { - if (err) { - var msg = err.message ? err.message : err; - return callback(new Error('could not fetch affected tables or last updated time: ' + msg)); - } - // when the table has not updated_at means it hasn't been changed so a default last_updated is set - var lastUpdated = 0; - if (rows.length !== 0) { - lastUpdated = rows[0].max || 0; - } - - return callback(null, lastUpdated*1000); - }); -}; function prepareSql(sql) { return sql diff --git a/lib/cartodb/backends/pg_connection.js b/lib/cartodb/backends/pg_connection.js index 2ca56f5b..d98bb703 100644 --- a/lib/cartodb/backends/pg_connection.js +++ b/lib/cartodb/backends/pg_connection.js @@ -1,5 +1,6 @@ var assert = require('assert'); var step = require('step'); +var PSQL = require('cartodb-psql'); var _ = require('underscore'); function PgConnection(metadataBackend) { @@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) { } ); }; + + +/** + * Returns a `cartodb-psql` object for a given username. + * @param {String} username + * @param {Function} callback function({Error}, {PSQL}) + */ + +PgConnection.prototype.getConnection = function(username, callback) { + var self = this; + + var params = {}; + + require('debug')('cachechan')("getConn1"); + step( + function setAuth() { + self.setDBAuth(username, params, this); + }, + function setConn(err) { + assert.ifError(err); + self.setDBConn(username, params, this); + }, + function openConnection(err) { + assert.ifError(err); + return callback(err, new PSQL({ + user: params.dbuser, + pass: params.dbpass, + host: params.dbhost, + port: params.dbport, + dbname: params.dbname + })); + } + ); +}; diff --git a/lib/cartodb/cache/model/database_tables_entry.js b/lib/cartodb/cache/model/database_tables_entry.js deleted file mode 100644 index a64c8dae..00000000 --- a/lib/cartodb/cache/model/database_tables_entry.js +++ /dev/null @@ -1,26 +0,0 @@ -var crypto = require('crypto'); - -function DatabaseTables(tables) { - this.namespace = 't'; - this.tables = tables; -} - -module.exports = DatabaseTables; - - -DatabaseTables.prototype.key = function() { - return this.tables.map(function(table) { - return this.namespace + ':' + shortHashKey(table.dbname + ':' + table.table_name + '.' + table.schema_name); - }.bind(this)).sort(); -}; - -DatabaseTables.prototype.getCacheChannel = function() { - var key = this.tables.map(function(table) { - return table.dbname + ':' + table.schema_name + "." + table.table_name; - }).sort().join(";;"); - return key; -}; - -function shortHashKey(target) { - return crypto.createHash('sha256').update(target).digest('base64').substring(0,6); -} diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index e647a046..6d393b27 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -8,7 +8,8 @@ var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); -var TablesCacheEntry = require('../cache/model/database_tables_entry'); + +var QueryTables = require('node-cartodb-query-tables'); /** * @param {AuthApi} authApi @@ -28,6 +29,7 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) { BaseController.call(this, authApi, pgConnection); + this.pgConnection = pgConnection; this.mapStore = mapStore; this.tileBackend = tileBackend; this.previewBackend = previewBackend; @@ -320,9 +322,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h global.logger.warn('ERROR generating cache channel: ' + err); } if (!!affectedTables) { - var tablesCacheEntry = new TablesCacheEntry(affectedTables); - res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); - self.surrogateKeysCache.tag(res, tablesCacheEntry); + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + self.surrogateKeysCache.tag(res, affectedTables); } self.send(req, res, body, status, headers); } @@ -366,11 +367,21 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg throw new Error("this request doesn't need an X-Cache-Channel generated"); } - self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel + step( + function getConnection() { + self.pgConnection.getConnection(user, this); + }, + function getAffectedTables(err, connection) { + assert.ifError(err); + + QueryTables.getAffectedTablesFromQuery(connection, sql, this); + }, + this + ); }, function buildCacheChannel(err, tables) { assert.ifError(err); - self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables); + self.layergroupAffectedTables.set(dbName, layergroupId, tables); return tables; }, @@ -378,7 +389,7 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg if(tables === undefined){ callback(err); }else{ - callback(err, tables.affectedTables); + callback(err, tables); } } ); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index a7613c2d..87fa85ea 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var step = require('step'); var windshaft = require('windshaft'); +var QueryTables = require('node-cartodb-query-tables'); var util = require('util'); var BaseController = require('./base'); @@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig; var Datasource = windshaft.model.Datasource; var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -var TablesCacheEntry = require('../cache/model/database_tables_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); @@ -329,31 +329,39 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la // return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime }); // }); // } else { - self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); + // self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); //} + step( + function getConnection() { + self.pgConnection.getConnection(username, this); + }, + function getAffectedTables(err, connection) { + assert.ifError(err); + QueryTables.getAffectedTablesFromQuery(connection, sql, this); + }, + this + ); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { if (req.profiler) { req.profiler.done('queryTablesAndLastUpdated'); } assert.ifError(err); - self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables); + self.layergroupAffectedTables.set(dbName, layergroupId, result); // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; - layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString(); + layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt(); + layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString(); // TODO this should take into account several URL patterns addWidgetsUrl(username, layergroup); - if (req.method === 'GET') { - var tableCacheEntry = new TablesCacheEntry(result.affectedTables); var ttl = global.environment.varnish.layergroupTtl || 86400; res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); res.set('Last-Modified', (new Date()).toUTCString()); - res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel()); - if (result.affectedTables && result.affectedTables.length > 0) { - self.surrogateKeysCache.tag(res, tableCacheEntry); + res.set('X-Cache-Channel', result.getCacheChannel()); + if (result.tables && result.tables.length > 0) { + self.surrogateKeysCache.tag(res, result); } } diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 3687dc39..38ae91e8 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -9,8 +9,6 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); -var TablesCacheEntry = require('../cache/model/database_tables_entry'); - function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { BaseController.call(this, authApi, pgConnection); @@ -53,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header if (err) { global.logger.log('ERROR generating cache channel: ' + err); } - if (!result || !!result.affectedTables) { + if (!result || !!result.tables) { // we increase cache control as we can invalidate it res.set('Cache-Control', 'public,max-age=31536000'); var lastModifiedDate; if (Number.isFinite(result.lastUpdatedTime)) { - lastModifiedDate = new Date(result.lastUpdatedTime); + lastModifiedDate = new Date(result.getLastUpdatedAt()); } else { lastModifiedDate = new Date(); } res.set('Last-Modified', lastModifiedDate.toUTCString()); - var tablesCacheEntry = new TablesCacheEntry(result.affectedTables); - res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); - if (result.affectedTables.length > 0) { - self.surrogateKeysCache.tag(res, tablesCacheEntry); + res.set('X-Cache-Channel', result.getCacheChannel()); + if (result.tables.length > 0) { + self.surrogateKeysCache.tag(res, result); } } self.send(req, res, resource, 200); @@ -230,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap return next(null); } - var affectedTables = affectedTablesAndLastUpdate.affectedTables || []; + var affectedTables = affectedTablesAndLastUpdate.tables || []; if (affectedTables.length === 0) { return next(null); diff --git a/package.json b/package.json index e31e26ff..d9877bb2 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "lru-cache": "2.6.5", "lzma": "~1.3.7", "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb" + "node-cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" }, "devDependencies": { "istanbul": "~0.3.6", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index d4e06e52..202fb259 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry'); +var QueryTables = require('node-cartodb-query-tables'); ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) { @@ -262,7 +262,7 @@ describe(suiteName, function() { var parsedBody = JSON.parse(res.body); expected_token = parsedBody.layergroupid.split(':')[0]; helper.checkCache(res); - helper.checkSurrogateKey(res, new TablesCacheEntry([ + helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([ {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"}, {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"}, ]).key().join(' ')); diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index 544aea6c..2c3801e0 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -7,6 +7,7 @@ var _ = require('underscore'); var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); +var QueryTables = require('node-cartodb-query-tables'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); @@ -309,6 +310,7 @@ describe('tests from old api translated to multilayer', function() { it("creates layergroup fails when postgresql queries fail to figure affected tables in query", function(done) { + var runQueryFn = PgQueryRunner.prototype.run; PgQueryRunner.prototype.run = function(username, query, callback) { return callback(new Error('fake error message'), []); @@ -343,6 +345,7 @@ describe('tests from old api translated to multilayer', function() { }); it("tile requests works when postgresql queries fail to figure affected tables in query", function(done) { + var layergroup = singleLayergroupConfig('select * from gadm4', '#gadm4 { marker-fill: red; }'); assert.response(server, { @@ -360,9 +363,11 @@ describe('tests from old api translated to multilayer', function() { keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0; keysToDelete['user:localhost:mapviews:global'] = 5; - var runQueryFn = PgQueryRunner.prototype.run; - PgQueryRunner.prototype.run = function(username, query, callback) { - return callback(new Error('failed to query database for affected tables'), []); + var affectedFn = QueryTables.getAffectedTablesFromQuery; + QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) { + affectedFn({query: function(query, callback) { + return callback(new Error('fake error message'), []); + }}, username, query, callback); }; // reset internal cacheChannel cache @@ -387,7 +392,7 @@ describe('tests from old api translated to multilayer', function() { }, function(res) { assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); - PgQueryRunner.prototype.run = runQueryFn; + QueryTables.getAffectedTablesFromQuery = affectedFn; done(); } ); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 137f5ce5..4b53e0a6 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -3,8 +3,8 @@ var _ = require('underscore'); var redis = require('redis'); var step = require('step'); var strftime = require('strftime'); +var QueryTables = require('node-cartodb-query-tables'); var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry'); -var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry'); var redis_stats_db = 5; // Pollute the PG environment to make sure @@ -1393,7 +1393,7 @@ describe('template_api', function() { // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); var expectedSurrogateKey = [ - new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public', + new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public', table_name: 'test_table_private_1'}]).key(), new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key() ].join(' '); @@ -1477,7 +1477,7 @@ describe('template_api', function() { // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); var expectedSurrogateKey = [ - new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public', + new QueryTables.DatabaseTableEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public', table_name: 'test_table_private_1'}]).key(), new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key() ].join(' ');