cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use new CDB_QueryTablesUpdatedAt function

Browse Source
This commit is contained in:
Alejandro Martínez 2016-02-09 19:06:34 +01:00
parent f0af107ffa
commit 95ab99be4d
7 changed files with 49 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/cartodb/api/query_tables_api.js
View File

@ -29,32 +29,26 @@ QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, cal
}; };
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) { QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
var query = [ var query =
'WITH querytables AS (', 'SELECT * FROM CDB_QueryTablesUpdatedAt($windshaft$' + prepareSql(sql) + '$windshaft$)';
'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
')',
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m',
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) { this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
if (err || rows.length === 0) { if (err) {
var msg = err.message ? err.message : err; var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables or last updated time: ' + msg)); callback(new Error('could not fetch affected tables or last updated time: ' + msg));
return; return;
} }
var result = rows[0]; var affectedTables = rows;
// This is an Array, so no need to split into parts var updatedTimes = affectedTables.map(function getUpdateDate(table) {
var tableNames = result.tablenames; return table.updated_at;
});
var lastUpdatedTime = result.max || 0; var lastUpdatedTime = (affectedTables.length === 0 ? 0 : Math.max.apply(null, updatedTimes)) || 0;
callback(null, { callback(null, {
affectedTables: tableNames, affectedTables: affectedTables,
lastUpdatedTime: lastUpdatedTime * 1000 lastUpdatedTime: lastUpdatedTime
}); });
}); });
}; };

10
lib/cartodb/api/tables_extent_api.js
View File

@ -13,13 +13,9 @@ module.exports = TablesExtentApi;
* `table_name` format as valid input * `table_name` format as valid input
* @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north` * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
*/ */
TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) { TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
var estimatedExtentSQLs = tableNames.map(function(tableName) { var estimatedExtentSQLs = tables.map(function(table) {
var schemaTable = tableName.split('.'); return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
if (schemaTable.length > 1) {
return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')";
}
return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')";
}); });
var query = [ var query = [

14
lib/cartodb/cache/model/database_tables_entry.js vendored
View File

@ -1,22 +1,24 @@
var crypto = require('crypto'); var crypto = require('crypto');
function DatabaseTables(dbName, tableNames) { function DatabaseTables(tables) {
this.namespace = 't'; this.namespace = 't';
this.dbName = dbName; this.tables = tables;
this.tableNames = tableNames;
} }
module.exports = DatabaseTables; module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() { DatabaseTables.prototype.key = function() {
return this.tableNames.map(function(tableName) { return this.tables.map(function(table) {
return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName); return this.namespace + ':' + shortHashKey(table.db_name + ':' + table.table_name + '.' + table.schema_name);
}.bind(this)); }.bind(this));
}; };
DatabaseTables.prototype.getCacheChannel = function() { DatabaseTables.prototype.getCacheChannel = function() {
return this.dbName + ':' + this.tableNames.join(','); var key = this.tables.map(function(table) {
return table.db_name + ':' + table.schema_name + "." + table.table_name;
}).join(";;");
return key;
}; };
function shortHashKey(target) { function shortHashKey(target) {

19
lib/cartodb/controllers/layergroup.js
View File

@ -320,7 +320,7 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
global.logger.warn('ERROR generating cache channel: ' + err); global.logger.warn('ERROR generating cache channel: ' + err);
} }
if (!!affectedTables) { if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables); var tablesCacheEntry = new TablesCacheEntry(affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry); self.surrogateKeysCache.tag(res, tablesCacheEntry);
} }
@ -366,17 +366,20 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
throw new Error("this request doesn't need an X-Cache-Channel generated"); throw new Error("this request doesn't need an X-Cache-Channel generated");
} }
self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel
}, },
function buildCacheChannel(err, tableNames) { function buildCacheChannel(err, tables) {
assert.ifError(err); assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, tableNames); return tables;
return tableNames;
}, },
function finish(err, affectedTables) { function finish(err, tables) {
callback(err, affectedTables); if(tables === undefined){
callback(err);
}else{
callback(err, tables.affectedTables);
}
} }
); );
}; };

26
lib/cartodb/controllers/map.js
View File

@ -280,20 +280,20 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
function checkCachedAffectedTables() { function checkCachedAffectedTables() {
return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId); return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
}, },
function getAffectedTablesAndLastUpdatedTime(err, hasCache) { function getAffectedTablesAndLastUpdatedTime(err) {
assert.ifError(err); assert.ifError(err);
if (hasCache) { // if (hasCache) {
var next = this; // var next = this;
var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId); // var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) { // self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
if (err) { // if (err) {
return next(err); // return next(err);
} // }
return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime }); // return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
}); // });
} else { // } else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
} //}
}, },
function handleAffectedTablesAndLastUpdatedTime(err, result) { function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) { if (req.profiler) {
@ -310,7 +310,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
addWidgetsUrl(username, layergroup); addWidgetsUrl(username, layergroup);
if (req.method === 'GET') { if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); var tableCacheEntry = new TablesCacheEntry(result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400; var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.set('Last-Modified', (new Date()).toUTCString()); res.set('Last-Modified', (new Date()).toUTCString());

3
lib/cartodb/controllers/named_maps.js
View File

@ -44,7 +44,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
var self = this; var self = this;
var dbName = req.params.dbname;
step( step(
function getAffectedTablesAndLastUpdatedTime() { function getAffectedTablesAndLastUpdatedTime() {
namedMapProvider.getAffectedTablesAndLastUpdatedTime(this); namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
@ -66,7 +65,7 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
} }
res.set('Last-Modified', lastModifiedDate.toUTCString()); res.set('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); var tablesCacheEntry = new TablesCacheEntry(result.affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel()); res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) { if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry); self.surrogateKeysCache.tag(res, tablesCacheEntry);

6
test/acceptance/multilayer.js
View File

@ -262,9 +262,9 @@ describe(suiteName, function() {
var parsedBody = JSON.parse(res.body); var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0]; expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res); helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [ helper.checkSurrogateKey(res, new TablesCacheEntry([
'public.test_table', {db_name: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
'public.test_table_2' {db_name: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
]).key().join(' ')); ]).key().join(' '));