cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use new CDB_QueryTablesUpdatedAt function

Browse Source
This commit is contained in:
Alejandro Martínez 2016-02-09 19:06:34 +01:00
parent f0af107ffa
commit 95ab99be4d
7 changed files with 49 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/cartodb/api/query_tables_api.js
View File

@ -29,32 +29,26 @@ QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, cal
};
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
var query = [
'WITH querytables AS (',
'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
')',
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m',
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
var query =
'SELECT * FROM CDB_QueryTablesUpdatedAt($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
if (err || rows.length === 0) {
if (err) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables or last updated time: ' + msg));
return;
}
var result = rows[0];
var affectedTables = rows;
// This is an Array, so no need to split into parts
var tableNames = result.tablenames;
var lastUpdatedTime = result.max || 0;
var updatedTimes = affectedTables.map(function getUpdateDate(table) {
return table.updated_at;
});
var lastUpdatedTime = (affectedTables.length === 0 ? 0 : Math.max.apply(null, updatedTimes)) || 0;
callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime * 1000
affectedTables: affectedTables,
lastUpdatedTime: lastUpdatedTime
});
});
};

10
lib/cartodb/api/tables_extent_api.js
View File

@ -13,13 +13,9 @@ module.exports = TablesExtentApi;
* `table_name` format as valid input
* @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
*/
TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) {
var estimatedExtentSQLs = tableNames.map(function(tableName) {
var schemaTable = tableName.split('.');
if (schemaTable.length > 1) {
return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')";
}
return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')";
TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
var estimatedExtentSQLs = tables.map(function(table) {
return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
});
var query = [

14
lib/cartodb/cache/model/database_tables_entry.js vendored
View File

@ -1,22 +1,24 @@
var crypto = require('crypto');
function DatabaseTables(dbName, tableNames) {
function DatabaseTables(tables) {
this.namespace = 't';
this.dbName = dbName;
this.tableNames = tableNames;
this.tables = tables;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tableNames.map(function(tableName) {
return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
return this.tables.map(function(table) {
return this.namespace + ':' + shortHashKey(table.db_name + ':' + table.table_name + '.' + table.schema_name);
}.bind(this));
};
DatabaseTables.prototype.getCacheChannel = function() {
return this.dbName + ':' + this.tableNames.join(',');
var key = this.tables.map(function(table) {
return table.db_name + ':' + table.schema_name + "." + table.table_name;
}).join(";;");
return key;
};
function shortHashKey(target) {

19
lib/cartodb/controllers/layergroup.js
View File

@ -320,7 +320,7 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
global.logger.warn('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
var tablesCacheEntry = new TablesCacheEntry(affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
}
@ -366,17 +366,20 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
function buildCacheChannel(err, tables) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
return tableNames;
return tables;
},
function finish(err, affectedTables) {
callback(err, affectedTables);
function finish(err, tables) {
if(tables === undefined){
callback(err);
}else{
callback(err, tables.affectedTables);
}
}
);
};

26
lib/cartodb/controllers/map.js
View File

@ -280,20 +280,20 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
function checkCachedAffectedTables() {
return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
},
function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
function getAffectedTablesAndLastUpdatedTime(err) {
assert.ifError(err);
if (hasCache) {
var next = this;
var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
if (err) {
return next(err);
}
return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
});
} else {
// if (hasCache) {
// var next = this;
// var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
// self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
// if (err) {
// return next(err);
// }
// return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
// });
// } else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
}
//}
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
@ -310,7 +310,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
var tableCacheEntry = new TablesCacheEntry(result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.set('Last-Modified', (new Date()).toUTCString());

3
lib/cartodb/controllers/named_maps.js
View File

@ -44,7 +44,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
var self = this;
var dbName = req.params.dbname;
step(
function getAffectedTablesAndLastUpdatedTime() {
namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
@ -66,7 +65,7 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
var tablesCacheEntry = new TablesCacheEntry(result.affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);

6
test/acceptance/multilayer.js
View File

@ -262,9 +262,9 @@ describe(suiteName, function() {
var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
'public.test_table',
'public.test_table_2'
helper.checkSurrogateKey(res, new TablesCacheEntry([
{db_name: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
{db_name: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
]).key().join(' '));