Multilayer API changes, target 1.2.0

- Layers passed by index in grid fetching url
 - Interactivity only specified in layergroup config
 - Encode cache_buster as part of the token

NEWS.md

@@ -1,17 +1,19 @@
-1.1.11
+1.2.0
 ------
-
+* Multilayer API changes
+  * Layers passed by index in grid fetching url
+  * Interactivity only specified in layergroup config
+  * Embed cache_buster within token 
+  * Use ISO format for last_modified timestamp
 
 1.1.10
 ------
-
 * Fix regression with default interactivity parameter (#74)
 * More verbose logging for SQL api connection errors
 * Write stats for multilayer map token request
 
 1.1.9
 -----
-
 * Handle SQL API errors by requesting no Varnish cache
 * Fix X-Cache-Channel for multilayer (by token) responses
 * Add last_modified field to layergroup creation response (#72)

lib/cartodb/server_options.js

@@ -263,7 +263,8 @@ module.exports = function(){
             // find last updated
             me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
               if ( err ) { done(err); return; }
-              response.last_updated = lastUpdated;
+              response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
+              response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format
               done(null);
             });
         });
@@ -322,6 +323,13 @@ module.exports = function(){
         _.each(bad_query, function(key){ delete req.query[key]; });
         req.params =  _.extend({}, req.params); // shuffle things as request is a strange array/object
 
+        if ( req.params.token ) {
+          //console.log("Request parameters include token " + req.params.token);
+          var tksplit = req.params.token.split(':');
+          req.params.token = tksplit[0];
+          if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1];
+        }
+
         // bring all query values onto req.params object
         _.extend(req.params, req.query);
 

npm-shrinkwrap.json

@@ -193,10 +193,10 @@
       "version": "1.3.3"
     },
     "windshaft": {
-      "version": "0.11.2",
+      "version": "0.12.0",
       "dependencies": {
         "grainstore": {
-          "version": "0.12.0",
+          "version": "0.13.0",
           "dependencies": {
             "carto": {
               "version": "0.9.3-cdb3",
@@ -288,7 +288,7 @@
           }
         },
         "tilelive": {
-          "version": "4.3.3",
+          "version": "4.4.2",
           "dependencies": {
             "optimist": {
               "version": "0.3.7",
@@ -307,9 +307,6 @@
           "version": "0.5.0",
           "from": "git://github.com/Vizzuality/tilelive-mapnik.git#6a360ee50",
           "dependencies": {
-            "generic-pool": {
-              "version": "2.0.3"
-            },
             "eio": {
               "version": "0.1.0"
             },
@@ -327,7 +324,7 @@
       "version": "0.0.5"
     },
     "generic-pool": {
-      "version": "1.0.12"
+      "version": "2.0.3"
     },
     "redis": {
       "version": "0.7.2"

package.json

@@ -21,9 +21,9 @@
         "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
         "node-varnish": "0.1.1",
         "underscore"  : "~1.3.3",
-        "windshaft"  : "~0.11.2",
+        "windshaft"  : "~0.12.0",
         "step": "0.0.x",
-        "generic-pool": "~1.0.12",
+        "generic-pool": "~2.0.3",
         "redis": "0.7.2",
         "hiredis": "~0.1.14",
         "request": "2.9.202",

test/acceptance/multilayer.js

@@ -21,6 +21,8 @@ suite('multilayer', function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
+    var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu
+    var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
 
     suiteSetup(function(done){
       sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
@@ -34,17 +36,19 @@ suite('multilayer', function() {
            { options: {
                sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
                cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
-               cartocss_version: '2.0.1' 
+               cartocss_version: '2.0.1',
+               interactivity: 'cartodb_id'
              } },
            { options: {
                sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2',
                cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
-               cartocss_version: '2.0.2' 
+               cartocss_version: '2.0.2',
+               interactivity: 'cartodb_id'
              } }
         ]
       };
 
-      var expected_token = "d55208dccb30b5ff972562f563db0d22";
+      var expected_token = "e34dd7e235138a062f8ba7ad051aa3a7";
       Step(
         function do_post()
         {
@@ -65,15 +69,9 @@ suite('multilayer', function() {
                     + layergroup.layers[1].options.sql 
                     + '$windshaft$)'
               });
-              expectedBody.last_updated = JSON.stringify({
-                'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max '
-                    + 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
-                    + qTables + '}\')'
-              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
               if ( expected_token ) {
-                //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
-                //assert.equal(parsedBody.last_updated, expectedBody.last_updated);
-                assert.deepEqual(parsedBody, expectedBody);
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
               }
               else expected_token = parsedBody.layergroupid;
               next(null, res);
@@ -84,7 +82,7 @@ suite('multilayer', function() {
           if ( err ) throw err;
           var next = this;
           assert.response(server, {
-              url: '/tiles/layergroup/' + expected_token + '/0/0/0.png',
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png',
               method: 'GET',
               headers: {host: 'localhost' },
               encoding: 'binary'
@@ -116,7 +114,7 @@ suite('multilayer', function() {
           var next = this;
           assert.response(server, {
               url: '/tiles/layergroup/' + expected_token
-                 + '/layer0/0/0/0.grid.json?interactivity=cartodb_id',
+                 + '/0/0/0/0.grid.json',
               headers: {host: 'localhost' },
               method: 'GET'
           }, {}, function(res) {
@@ -134,7 +132,7 @@ suite('multilayer', function() {
           var next = this;
           assert.response(server, {
               url: '/tiles/layergroup/' + expected_token
-                 + '/layer1/0/0/0.grid.json?interactivity=cartodb_id',
+                 + '/1/0/0/0.grid.json',
               headers: {host: 'localhost' },
               method: 'GET'
           }, {}, function(res) {
@@ -175,12 +173,13 @@ suite('multilayer', function() {
                sql: 'select 1 as cartodb_id, '
                   + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
                cartocss: '#layer { polygon-fill:red; }', 
-               cartocss_version: '2.0.1' 
+               cartocss_version: '2.0.1',
+               interactivity: 'cartodb_id'
              } }
         ]
       };
 
-      var expected_token  = "20f5710c00e3a1b0b4950de65ef0d875";
+      var expected_token  = "6d8e4ad5458e2d25cf0eef38e38717a6";
       Step(
         function do_post()
         {
@@ -200,15 +199,9 @@ suite('multilayer', function() {
                     + layergroup.layers[0].options.sql
                     + '$windshaft$)'
               });
-              expectedBody.last_updated = JSON.stringify({
-                'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max '
-                    + 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
-                    + qTables + '}\')'
-              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
               if ( expected_token ) {
-                //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
-                //assert.equal(parsedBody.last_updated, expectedBody.last_updated);
-                assert.deepEqual(parsedBody, expectedBody);
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
               }
               else expected_token = parsedBody.layergroupid;
               next(null, res);
@@ -219,7 +212,7 @@ suite('multilayer', function() {
           if ( err ) throw err;
           var next = this;
           assert.response(server, {
-              url: '/tiles/layergroup/' + expected_token + '/1/0/0.png',
+              url: '/tiles/layergroup/' + expected_token + ':cb10/1/0/0.png',
               method: 'GET',
               headers: {host: 'localhost' },
               encoding: 'binary'
@@ -249,7 +242,7 @@ suite('multilayer', function() {
           if ( err ) throw err;
           var next = this;
           assert.response(server, {
-              url: '/tiles/layergroup/' + expected_token + '/4/0/0.png',
+              url: '/tiles/layergroup/' + expected_token + ':cb11/4/0/0.png',
               method: 'GET',
               headers: {host: 'localhost' },
               encoding: 'binary'
@@ -280,7 +273,7 @@ suite('multilayer', function() {
           var next = this;
           assert.response(server, {
               url: '/tiles/layergroup/' + expected_token
-                 + '/layer0/1/0/0.grid.json?interactivity=cartodb_id',
+                 + '/0/1/0/0.grid.json',
               headers: {host: 'localhost' },
               method: 'GET'
           }, {}, function(res) {
@@ -298,7 +291,7 @@ suite('multilayer', function() {
           var next = this;
           assert.response(server, {
               url: '/tiles/layergroup/' + expected_token
-                 + '/layer0/4/0/0.grid.json?interactivity=cartodb_id',
+                 + '/0/4/0/0.grid.json',
               headers: {host: 'localhost' },
               method: 'GET'
           }, {}, function(res) {
@@ -397,6 +390,8 @@ suite('multilayer', function() {
             errors.push(err.message);
             console.log("Error: " + err);
           }
+          // trip epoch
+          expected_token = expected_token.split(':')[0];
           redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
               if ( err ) errors.push(err.message);
               assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);

test/support/SQLAPIEmu.js

@@ -3,11 +3,20 @@ var url         = require('url');
 
 var o = function(port, cb) {
 
+  this.queries = [];
+  var that = this;
   this.sqlapi_server = http.createServer(function(req,res) {
         var query = url.parse(req.url, true).query;
+        that.queries.push(query);
         if ( query.q.match('SQLAPIERROR') ) {
           res.statusCode = 400;
           res.write(JSON.stringify({'error':'Some error occurred'}));
+        } else if ( query.q.match('EPOCH.* as max') ) {
+          // This is the structure of the known query sent by tiler
+          var row = {
+            'max': 1234567890123
+          };
+          res.write(JSON.stringify({rows: [ row ]}));
         } else {
           var qs = JSON.stringify(query);
           var row = {