Add last_modified field to POST layergroup response (#72)

Includes testcases

NEWS.md

@@ -2,6 +2,7 @@
 -----
 * Handle SQL API errors by requesting no Varnish cache
 * Fix X-Cache-Channel for multilayer (by token) responses
+* Add last_modified field to POST layergroup response (#72)
 
 1.1.8
 -----

lib/cartodb/server_options.js

@@ -54,42 +54,70 @@ module.exports = function(){
     //       we have no SQL after layer creation.
     me.channelCache = {};
 
-    me.affectedTables = function (username, api_key, sql, callback) {
+    // Run a query through the SQL api
-
+    me.sqlQuery = function (username, api_key, sql, callback) {
       var api = global.environment.sqlapi;
 
       // build up api string
       var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
 
-      var qs  = {};
+      var qs  = { q: sql }
-
-      // add query to querystring
-      qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
 
       // add api_key if given
       if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
 
       // call sql api
       request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
-          var epref = 'could not detect source tables using SQL api at ' + sqlapi;
           if (err){
-            var msg = err.message ? err.message : err;
+            callback(err);
-            callback(new Error(epref + ': ' + msg));
             return;
           }
           if (res.statusCode != 200) {
               var msg = res.body.error ? res.body.error : res.body;
-              callback(new Error(epref + ': ' + msg));
+              callback(new Error('unexpected response status (' + res.statusCode + ') for sql query: ' + sql));
               return;
           } 
-          var qtables = body.rows[0].cdb_querytables;
+          callback(null, body.rows);
+      });
+    };
+
+    me.findLastUpdated = function (username, api_key, tableNames, callback) {
+
+      var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) FROM CDB_TableMetadata WHERE m.tabname::name = any ({'
+        + tableNames.join(',') + '})';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not find last updated timestamp: ' + msg));
+            return;
+          }
+          var last_updated = rows[0].max;
+          callback(null, last_updated);
+      });
+    };
+
+    me.affectedTables = function (username, api_key, sql, callback) {
+
+      var sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not fetch source tables: ' + msg));
+            return;
+          }
+          var qtables = rows[0].cdb_querytables;
           var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+          tableNames = tableNames.split(',');
           callback(null, tableNames);
       });
-    },
+    };
 
     me.buildCacheChannel = function (dbName, tableNames){
-      return dbName + ':' + tableNames;
+      return dbName + ':' + tableNames.join(',');
     };
 
     me.generateMD5 = function(data){
@@ -124,14 +152,13 @@ module.exports = function(){
       }
 
       if ( ! req.params.sql && ! req.params.token ) {
-        var cacheChannel = me.buildCacheChannel(dbName, req.params.table);
+        var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
         // not worth caching this
         callback(null, cacheChannel);
         return;
       }
 
       if ( ! req.params.sql ) {
-console.log('req:'); console.dir(req);
         callback(new Error("this request doesn't need an X-Cache-Channel generated"));
         return;
       }
@@ -194,7 +221,6 @@ console.log('req:'); console.dir(req);
           sql.push(lyr.options.sql);
         });
         sql = sql.join(';');
-        console.log('afterLayergroupCreate: sql:'+sql);
 
         var dbName = req.params.dbname;
         var usr    = req.headers.host.split('.')[0];
@@ -203,11 +229,17 @@ console.log('req:'); console.dir(req);
         var cacheKey = dbName + ':' + token;
 
         me.affectedTables(usr, key, sql, function(err, tableNames) {
+
             if ( err ) { callback(err); return; }
             var cacheChannel = me.buildCacheChannel(dbName,tableNames);
             me.channelCache[cacheKey] = cacheChannel; // store for caching
+            // find last updated
+            me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
+              if ( err ) { callback(err); return; }
+              response.last_updated = lastUpdated;
               callback(null);
             });
+        });
     };
 
 /* X-Cache-Channel generation } */

test/acceptance/multilayer.js

@@ -6,8 +6,7 @@ var querystring = require('querystring');
 var semver      = require('semver');
 var mapnik      = require('mapnik');
 var Step        = require('step');
-var http        = require('http');
+var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
-var url         = require('url');
 
 require(__dirname + '/../support/test_helper');
 
@@ -24,18 +23,7 @@ suite('multilayer', function() {
     var sqlapi_server;
 
     suiteSetup(function(done){
-      sqlapi_server = http.createServer(function(req,res) {
+      sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
-        var query = url.parse(req.url, true).query;
-        if ( query.q.match('SQLAPIERROR') ) {
-          res.statusCode = 400;
-          res.write(JSON.stringify({'error':'Some error occurred'}));
-        } else {
-          res.write(JSON.stringify({rows: [ { 'cdb_querytables': '{' +
-            JSON.stringify(query) + '}' } ]}));
-        }
-        res.end();
-      });
-      sqlapi_server.listen(global.environment.sqlapi.port, done);
     });
 
     test("layergroup with 2 layers, each with its style", function(done) {
@@ -70,9 +58,23 @@ suite('multilayer', function() {
               assert.equal(res.statusCode, 200, res.body);
               var parsedBody = JSON.parse(res.body);
               var expectedBody = { layergroupid: expected_token };
-              // TODO: check last modified
+              // check last modified
-              //expectedBody.layercount = 2;
+              var qTables = JSON.stringify({
-              if ( expected_token ) assert.deepEqual(parsedBody, expectedBody);
+                'q': 'SELECT CDB_QueryTables($windshaft$'
+                    + layergroup.layers[0].options.sql + ';'
+                    + layergroup.layers[1].options.sql 
+                    + '$windshaft$)'
+              });
+              expectedBody.last_updated = JSON.stringify({
+                'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) '
+                    + 'FROM CDB_TableMetadata WHERE m.tabname::name = any ({'
+                    + qTables + '})'
+              });
+              if ( expected_token ) {
+                //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
+                //assert.equal(parsedBody.last_updated, expectedBody.last_updated);
+                assert.deepEqual(parsedBody, expectedBody);
+              }
               else expected_token = parsedBody.layergroupid;
               next(null, res);
           });

test/acceptance/server.js

@@ -7,7 +7,7 @@ var semver      = require('semver');
 var mapnik      = require('mapnik');
 var Step        = require('step');
 var http        = require('http');
-var url         = require('url');
+var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
 
 require(__dirname + '/../support/test_helper');
 
@@ -34,18 +34,7 @@ suite('server', function() {
     var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
     
     suiteSetup(function(done){
-      sqlapi_server = http.createServer(function(req,res) {
+      sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
-        var query = url.parse(req.url, true).query;
-        if ( query.q.match('SQLAPIERROR') ) {
-          res.statusCode = 400;
-          res.write(JSON.stringify({'error':'Some error occurred'}));
-        } else {
-          res.write(JSON.stringify({rows: [ { 'cdb_querytables': '{' +
-            JSON.stringify(query) + '}' } ]}));
-        }
-        res.end();
-      });
-      sqlapi_server.listen(global.environment.sqlapi.port, done);
     });
 
     /////////////////////////////////////////////////////////////////////////////////

test/support/SQLAPIEmu.js

@@ -0,0 +1,29 @@
+var http        = require('http');
+var url         = require('url');
+
+var o = function(port, cb) {
+
+  this.sqlapi_server = http.createServer(function(req,res) {
+        var query = url.parse(req.url, true).query;
+        if ( query.q.match('SQLAPIERROR') ) {
+          res.statusCode = 400;
+          res.write(JSON.stringify({'error':'Some error occurred'}));
+        } else {
+          var qs = JSON.stringify(query);
+          var row = {
+            // This is the structure of the known query sent by tiler
+            'cdb_querytables': '{' + qs + '}',
+            'max': qs
+          };
+          res.write(JSON.stringify({rows: [ row ]}));
+        }
+        res.end();
+   }).listen(port, cb);
+};
+
+o.prototype.close = function(cb) {
+  this.sqlapi_server.close(cb);
+};
+
+module.exports = o;
+