From 4605bd1e1d3ade37772c4209cba6080e38c7f13a Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 13 Mar 2013 18:41:37 +0100 Subject: [PATCH] Add last_modified field to POST layergroup response (#72) Includes testcases --- NEWS.md | 1 + lib/cartodb/server_options.js | 68 +++++++++++++++++++++++++---------- test/acceptance/multilayer.js | 36 ++++++++++--------- test/acceptance/server.js | 15 ++------ test/support/SQLAPIEmu.js | 29 +++++++++++++++ 5 files changed, 101 insertions(+), 48 deletions(-) create mode 100644 test/support/SQLAPIEmu.js diff --git a/NEWS.md b/NEWS.md index 14bd722c..e496be88 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ----- * Handle SQL API errors by requesting no Varnish cache * Fix X-Cache-Channel for multilayer (by token) responses +* Add last_modified field to POST layergroup response (#72) 1.1.8 ----- diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 328b2e87..aaa58fea 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -54,42 +54,70 @@ module.exports = function(){ // we have no SQL after layer creation. me.channelCache = {}; - me.affectedTables = function (username, api_key, sql, callback) { - + // Run a query through the SQL api + me.sqlQuery = function (username, api_key, sql, callback) { var api = global.environment.sqlapi; // build up api string var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql' - var qs = {}; - - // add query to querystring - qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; + var qs = { q: sql } // add api_key if given - if (_.isString(api_key) && api_key != ''){ qs.api_key = api_key; } + if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; } // call sql api request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){ - var epref = 'could not detect source tables using SQL api at ' + sqlapi; if (err){ - var msg = err.message ? err.message : err; - callback(new Error(epref + ': ' + msg)); + callback(err); return; } if (res.statusCode != 200) { var msg = res.body.error ? res.body.error : res.body; - callback(new Error(epref + ': ' + msg)); + callback(new Error('unexpected response status (' + res.statusCode + ') for sql query: ' + sql)); return; } - var qtables = body.rows[0].cdb_querytables; + callback(null, body.rows); + }); + }; + + me.findLastUpdated = function (username, api_key, tableNames, callback) { + + var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) FROM CDB_TableMetadata WHERE m.tabname::name = any ({' + + tableNames.join(',') + '})'; + + // call sql api + me.sqlQuery(username, api_key, sql, function(err, rows){ + if (err){ + var msg = err.message ? err.message : err; + callback(new Error('could not find last updated timestamp: ' + msg)); + return; + } + var last_updated = rows[0].max; + callback(null, last_updated); + }); + }; + + me.affectedTables = function (username, api_key, sql, callback) { + + var sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; + + // call sql api + me.sqlQuery(username, api_key, sql, function(err, rows){ + if (err){ + var msg = err.message ? err.message : err; + callback(new Error('could not fetch source tables: ' + msg)); + return; + } + var qtables = rows[0].cdb_querytables; var tableNames = qtables.split(/^\{(.*)\}$/)[1]; + tableNames = tableNames.split(','); callback(null, tableNames); }); - }, + }; me.buildCacheChannel = function (dbName, tableNames){ - return dbName + ':' + tableNames; + return dbName + ':' + tableNames.join(','); }; me.generateMD5 = function(data){ @@ -124,14 +152,13 @@ module.exports = function(){ } if ( ! req.params.sql && ! req.params.token ) { - var cacheChannel = me.buildCacheChannel(dbName, req.params.table); + var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]); // not worth caching this callback(null, cacheChannel); return; } if ( ! req.params.sql ) { -console.log('req:'); console.dir(req); callback(new Error("this request doesn't need an X-Cache-Channel generated")); return; } @@ -194,7 +221,6 @@ console.log('req:'); console.dir(req); sql.push(lyr.options.sql); }); sql = sql.join(';'); - console.log('afterLayergroupCreate: sql:'+sql); var dbName = req.params.dbname; var usr = req.headers.host.split('.')[0]; @@ -203,10 +229,16 @@ console.log('req:'); console.dir(req); var cacheKey = dbName + ':' + token; me.affectedTables(usr, key, sql, function(err, tableNames) { + if ( err ) { callback(err); return; } var cacheChannel = me.buildCacheChannel(dbName,tableNames); me.channelCache[cacheKey] = cacheChannel; // store for caching - callback(null); + // find last updated + me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) { + if ( err ) { callback(err); return; } + response.last_updated = lastUpdated; + callback(null); + }); }); }; diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 2b1a27dd..601983df 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -6,8 +6,7 @@ var querystring = require('querystring'); var semver = require('semver'); var mapnik = require('mapnik'); var Step = require('step'); -var http = require('http'); -var url = require('url'); +var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); require(__dirname + '/../support/test_helper'); @@ -24,18 +23,7 @@ suite('multilayer', function() { var sqlapi_server; suiteSetup(function(done){ - sqlapi_server = http.createServer(function(req,res) { - var query = url.parse(req.url, true).query; - if ( query.q.match('SQLAPIERROR') ) { - res.statusCode = 400; - res.write(JSON.stringify({'error':'Some error occurred'})); - } else { - res.write(JSON.stringify({rows: [ { 'cdb_querytables': '{' + - JSON.stringify(query) + '}' } ]})); - } - res.end(); - }); - sqlapi_server.listen(global.environment.sqlapi.port, done); + sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); }); test("layergroup with 2 layers, each with its style", function(done) { @@ -70,9 +58,23 @@ suite('multilayer', function() { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); var expectedBody = { layergroupid: expected_token }; - // TODO: check last modified - //expectedBody.layercount = 2; - if ( expected_token ) assert.deepEqual(parsedBody, expectedBody); + // check last modified + var qTables = JSON.stringify({ + 'q': 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + ';' + + layergroup.layers[1].options.sql + + '$windshaft$)' + }); + expectedBody.last_updated = JSON.stringify({ + 'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) ' + + 'FROM CDB_TableMetadata WHERE m.tabname::name = any ({' + + qTables + '})' + }); + if ( expected_token ) { + //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid); + //assert.equal(parsedBody.last_updated, expectedBody.last_updated); + assert.deepEqual(parsedBody, expectedBody); + } else expected_token = parsedBody.layergroupid; next(null, res); }); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 4250c7de..33f8cc30 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -7,7 +7,7 @@ var semver = require('semver'); var mapnik = require('mapnik'); var Step = require('step'); var http = require('http'); -var url = require('url'); +var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); require(__dirname + '/../support/test_helper'); @@ -34,18 +34,7 @@ suite('server', function() { var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}"; suiteSetup(function(done){ - sqlapi_server = http.createServer(function(req,res) { - var query = url.parse(req.url, true).query; - if ( query.q.match('SQLAPIERROR') ) { - res.statusCode = 400; - res.write(JSON.stringify({'error':'Some error occurred'})); - } else { - res.write(JSON.stringify({rows: [ { 'cdb_querytables': '{' + - JSON.stringify(query) + '}' } ]})); - } - res.end(); - }); - sqlapi_server.listen(global.environment.sqlapi.port, done); + sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); }); ///////////////////////////////////////////////////////////////////////////////// diff --git a/test/support/SQLAPIEmu.js b/test/support/SQLAPIEmu.js new file mode 100644 index 00000000..68ab26db --- /dev/null +++ b/test/support/SQLAPIEmu.js @@ -0,0 +1,29 @@ +var http = require('http'); +var url = require('url'); + +var o = function(port, cb) { + + this.sqlapi_server = http.createServer(function(req,res) { + var query = url.parse(req.url, true).query; + if ( query.q.match('SQLAPIERROR') ) { + res.statusCode = 400; + res.write(JSON.stringify({'error':'Some error occurred'})); + } else { + var qs = JSON.stringify(query); + var row = { + // This is the structure of the known query sent by tiler + 'cdb_querytables': '{' + qs + '}', + 'max': qs + }; + res.write(JSON.stringify({rows: [ row ]})); + } + res.end(); + }).listen(port, cb); +}; + +o.prototype.close = function(cb) { + this.sqlapi_server.close(cb); +}; + +module.exports = o; +