cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add last_modified field to POST layergroup response (#72)

Browse Source 
Includes testcases
This commit is contained in:
Sandro Santilli 2013-03-13 18:41:37 +01:00
parent dfc4a02398
commit 4605bd1e1d
5 changed files with 101 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -2,6 +2,7 @@
-----
* Handle SQL API errors by requesting no Varnish cache
* Fix X-Cache-Channel for multilayer (by token) responses
* Add last_modified field to POST layergroup response (#72)
1.1.8
-----

68
lib/cartodb/server_options.js
View File

@ -54,42 +54,70 @@ module.exports = function(){
// we have no SQL after layer creation.
me.channelCache = {};
me.affectedTables = function (username, api_key, sql, callback) {
// Run a query through the SQL api
me.sqlQuery = function (username, api_key, sql, callback) {
var api = global.environment.sqlapi;
// build up api string
var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
var qs = {};
// add query to querystring
qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
var qs = { q: sql }
// add api_key if given
if (_.isString(api_key) && api_key != ''){ qs.api_key = api_key; }
if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
var epref = 'could not detect source tables using SQL api at ' + sqlapi;
if (err){
var msg = err.message ? err.message : err;
callback(new Error(epref + ': ' + msg));
callback(err);
return;
}
if (res.statusCode != 200) {
var msg = res.body.error ? res.body.error : res.body;
callback(new Error(epref + ': ' + msg));
callback(new Error('unexpected response status (' + res.statusCode + ') for sql query: ' + sql));
return;
}
var qtables = body.rows[0].cdb_querytables;
callback(null, body.rows);
});
};
me.findLastUpdated = function (username, api_key, tableNames, callback) {
var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) FROM CDB_TableMetadata WHERE m.tabname::name = any ({'
+ tableNames.join(',') + '})';
// call sql api
me.sqlQuery(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not find last updated timestamp: ' + msg));
return;
}
var last_updated = rows[0].max;
callback(null, last_updated);
});
};
me.affectedTables = function (username, api_key, sql, callback) {
var sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// call sql api
me.sqlQuery(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames.split(',');
callback(null, tableNames);
});
},
};
me.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames;
return dbName + ':' + tableNames.join(',');
};
me.generateMD5 = function(data){
@ -124,14 +152,13 @@ module.exports = function(){
}
if ( ! req.params.sql && ! req.params.token ) {
var cacheChannel = me.buildCacheChannel(dbName, req.params.table);
var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
// not worth caching this
callback(null, cacheChannel);
return;
}
if ( ! req.params.sql ) {
console.log('req:'); console.dir(req);
callback(new Error("this request doesn't need an X-Cache-Channel generated"));
return;
}
@ -194,7 +221,6 @@ console.log('req:'); console.dir(req);
sql.push(lyr.options.sql);
});
sql = sql.join(';');
console.log('afterLayergroupCreate: sql:'+sql);
var dbName = req.params.dbname;
var usr = req.headers.host.split('.')[0];
@ -203,10 +229,16 @@ console.log('req:'); console.dir(req);
var cacheKey = dbName + ':' + token;
me.affectedTables(usr, key, sql, function(err, tableNames) {
if ( err ) { callback(err); return; }
var cacheChannel = me.buildCacheChannel(dbName,tableNames);
me.channelCache[cacheKey] = cacheChannel; // store for caching
callback(null);
// find last updated
me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
if ( err ) { callback(err); return; }
response.last_updated = lastUpdated;
callback(null);
});
});
};

36
test/acceptance/multilayer.js
View File

@ -6,8 +6,7 @@ var querystring = require('querystring');
var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
var http = require('http');
var url = require('url');
var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js');
require(__dirname + '/../support/test_helper');
@ -24,18 +23,7 @@ suite('multilayer', function() {
var sqlapi_server;
suiteSetup(function(done){
sqlapi_server = http.createServer(function(req,res) {
var query = url.parse(req.url, true).query;
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else {
res.write(JSON.stringify({rows: [ { 'cdb_querytables': '{' +
JSON.stringify(query) + '}' } ]}));
}
res.end();
});
sqlapi_server.listen(global.environment.sqlapi.port, done);
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
});
test("layergroup with 2 layers, each with its style", function(done) {
@ -70,9 +58,23 @@ suite('multilayer', function() {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
var expectedBody = { layergroupid: expected_token };
// TODO: check last modified
//expectedBody.layercount = 2;
if ( expected_token ) assert.deepEqual(parsedBody, expectedBody);
// check last modified
var qTables = JSON.stringify({
'q': 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql + ';'
+ layergroup.layers[1].options.sql
+ '$windshaft$)'
});
expectedBody.last_updated = JSON.stringify({
'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) '
+ 'FROM CDB_TableMetadata WHERE m.tabname::name = any ({'
+ qTables + '})'
});
if ( expected_token ) {
//assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
//assert.equal(parsedBody.last_updated, expectedBody.last_updated);
assert.deepEqual(parsedBody, expectedBody);
}
else expected_token = parsedBody.layergroupid;
next(null, res);
});

15
test/acceptance/server.js
View File

@ -7,7 +7,7 @@ var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
var http = require('http');
var url = require('url');
var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js');
require(__dirname + '/../support/test_helper');
@ -34,18 +34,7 @@ suite('server', function() {
var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
suiteSetup(function(done){
sqlapi_server = http.createServer(function(req,res) {
var query = url.parse(req.url, true).query;
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else {
res.write(JSON.stringify({rows: [ { 'cdb_querytables': '{' +
JSON.stringify(query) + '}' } ]}));
}
res.end();
});
sqlapi_server.listen(global.environment.sqlapi.port, done);
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
});
/////////////////////////////////////////////////////////////////////////////////

29
test/support/SQLAPIEmu.js Normal file
View File

@ -0,0 +1,29 @@
var http = require('http');
var url = require('url');
var o = function(port, cb) {
this.sqlapi_server = http.createServer(function(req,res) {
var query = url.parse(req.url, true).query;
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else {
var qs = JSON.stringify(query);
var row = {
// This is the structure of the known query sent by tiler
'cdb_querytables': '{' + qs + '}',
'max': qs
};
res.write(JSON.stringify({rows: [ row ]}));
}
res.end();
}).listen(port, cb);
};
o.prototype.close = function(cb) {
this.sqlapi_server.close(cb);
};
module.exports = o;