cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use node-cartodb-query-tables library

Browse Source
This commit is contained in:
Alejandro Martínez 2016-02-22 11:40:25 +01:00
parent e1732076fc
commit cf06ff86c2
10 changed files with 92 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/cartodb/api/query_tables_api.js
View File

@ -53,32 +53,6 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
});
};
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, 0);
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
if (err) {
var msg = err.message ? err.message : err;
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var lastUpdated = 0;
if (rows.length !== 0) {
lastUpdated = rows[0].max || 0;
}
return callback(null, lastUpdated*1000);
});
};
function prepareSql(sql) {
return sql

35
lib/cartodb/backends/pg_connection.js
View File

@ -1,5 +1,6 @@
var assert = require('assert');
var step = require('step');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
function PgConnection(metadataBackend) {
@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
}
);
};
/**
* Returns a `cartodb-psql` object for a given username.
* @param {String} username
* @param {Function} callback function({Error}, {PSQL})
*/
PgConnection.prototype.getConnection = function(username, callback) {
var self = this;
var params = {};
require('debug')('cachechan')("getConn1");
step(
function setAuth() {
self.setDBAuth(username, params, this);
},
function setConn(err) {
assert.ifError(err);
self.setDBConn(username, params, this);
},
function openConnection(err) {
assert.ifError(err);
return callback(err, new PSQL({
user: params.dbuser,
pass: params.dbpass,
host: params.dbhost,
port: params.dbport,
dbname: params.dbname
}));
}
);
};

26
lib/cartodb/cache/model/database_tables_entry.js vendored
View File

@ -1,26 +0,0 @@
var crypto = require('crypto');
function DatabaseTables(tables) {
this.namespace = 't';
this.tables = tables;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tables.map(function(table) {
return this.namespace + ':' + shortHashKey(table.dbname + ':' + table.table_name + '.' + table.schema_name);
}.bind(this)).sort();
};
DatabaseTables.prototype.getCacheChannel = function() {
var key = this.tables.map(function(table) {
return table.dbname + ':' + table.schema_name + "." + table.table_name;
}).sort().join(";;");
return key;
};
function shortHashKey(target) {
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
}

25
lib/cartodb/controllers/layergroup.js
View File

@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var QueryTables = require('node-cartodb-query-tables');
/**
* @param {AuthApi} authApi
@ -28,6 +29,7 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
@ -320,9 +322,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
global.logger.warn('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
self.surrogateKeysCache.tag(res, affectedTables);
}
self.send(req, res, body, status, headers);
}
@ -366,11 +367,21 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel
step(
function getConnection() {
self.pgConnection.getConnection(user, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function buildCacheChannel(err, tables) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, tables);
return tables;
},
@ -378,7 +389,7 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
if(tables === undefined){
callback(err);
}else{
callback(err, tables.affectedTables);
callback(err, tables);
}
}
);

28
lib/cartodb/controllers/map.js
View File

@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var step = require('step');
var windshaft = require('windshaft');
var QueryTables = require('node-cartodb-query-tables');
var util = require('util');
var BaseController = require('./base');
@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
@ -329,31 +329,39 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
// return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
// });
// } else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
// self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
//}
step(
function getConnection() {
self.pgConnection.getConnection(username, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, result);
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
// TODO this should take into account several URL patterns
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.set('Last-Modified', (new Date()).toUTCString());
res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
if (result.affectedTables && result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tableCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables && result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}

15
lib/cartodb/controllers/named_maps.js
View File

@ -9,8 +9,6 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
BaseController.call(this, authApi, pgConnection);
@ -53,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
if (err) {
global.logger.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.affectedTables) {
if (!result || !!result.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.lastUpdatedTime);
lastModifiedDate = new Date(result.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(result.affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
self.send(req, res, resource, 200);
@ -230,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
return next(null);
}
var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
var affectedTables = affectedTablesAndLastUpdate.tables || [];
if (affectedTables.length === 0) {
return next(null);

1
package.json
View File

@ -37,6 +37,7 @@
"lru-cache": "2.6.5",
"lzma": "~1.3.7",
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
"node-cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
},
"devDependencies": {
"istanbul": "~0.3.6",

4
test/acceptance/multilayer.js
View File

@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var QueryTables = require('node-cartodb-query-tables');
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
@ -262,7 +262,7 @@ describe(suiteName, function() {
var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry([
helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
]).key().join(' '));

13
test/acceptance/multilayer_server.js
View File

@ -7,6 +7,7 @@ var _ = require('underscore');
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTables = require('node-cartodb-query-tables');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
@ -309,6 +310,7 @@ describe('tests from old api translated to multilayer', function() {
it("creates layergroup fails when postgresql queries fail to figure affected tables in query", function(done) {
var runQueryFn = PgQueryRunner.prototype.run;
PgQueryRunner.prototype.run = function(username, query, callback) {
return callback(new Error('fake error message'), []);
@ -343,6 +345,7 @@ describe('tests from old api translated to multilayer', function() {
});
it("tile requests works when postgresql queries fail to figure affected tables in query", function(done) {
var layergroup = singleLayergroupConfig('select * from gadm4', '#gadm4 { marker-fill: red; }');
assert.response(server,
{
@ -360,9 +363,11 @@ describe('tests from old api translated to multilayer', function() {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
var runQueryFn = PgQueryRunner.prototype.run;
PgQueryRunner.prototype.run = function(username, query, callback) {
return callback(new Error('failed to query database for affected tables'), []);
var affectedFn = QueryTables.getAffectedTablesFromQuery;
QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
affectedFn({query: function(query, callback) {
return callback(new Error('fake error message'), []);
}}, username, query, callback);
};
// reset internal cacheChannel cache
@ -387,7 +392,7 @@ describe('tests from old api translated to multilayer', function() {
},
function(res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
PgQueryRunner.prototype.run = runQueryFn;
QueryTables.getAffectedTablesFromQuery = affectedFn;
done();
}
);

6
test/acceptance/templates.js
View File

@ -3,8 +3,8 @@ var _ = require('underscore');
var redis = require('redis');
var step = require('step');
var strftime = require('strftime');
var QueryTables = require('node-cartodb-query-tables');
var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var redis_stats_db = 5;
// Pollute the PG environment to make sure
@ -1393,7 +1393,7 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');
@ -1477,7 +1477,7 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
new QueryTables.DatabaseTableEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');