Merge branch 'release/staging'

NEWS.md

@@ -0,0 +1,5 @@
+0.9.0 (25/09/12)
+-----
+* External resources in CartoCSS
+* Added X-Cache-Channel header in all the tiler GET requests
+* Small fixes

config/environments/development.js

@@ -18,6 +18,9 @@ var config = {
         */
         simplify: true
     }
+    ,millstone: {
+        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
+    }
     ,redis: {
         host: '127.0.0.1',
         port: 6379,

config/environments/production.js

@@ -12,6 +12,9 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,millstone: {
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
     ,redis: {
         host: '127.0.0.1',
         port: 6379

config/environments/staging.js

@@ -12,6 +12,9 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,millstone: {
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
     ,redis: {
         host: '127.0.0.1',
         port: 6379

config/environments/test.js

@@ -13,6 +13,9 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,millstone: {
+        cache_basedir: '/tmp/cdb-tiler-test/millstone'
+    }
     ,redis: {
         host: '127.0.0.1',
         port: 6333,

lib/cartodb/cartodb_windshaft.js

@@ -6,17 +6,6 @@ var _ = require('underscore')
 
 var CartodbWindshaft = function(serverOptions) {
 
-    // set the cache chanel info to invalidate the cache on the frontend server
-    serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
-        var ttl = global.environment.varnish.ttl || 86400;
-        Cache.generateCacheChannel(req, function(channel){
-            res.header('X-Cache-Channel', channel);
-            res.header('Last-Modified', new Date().toUTCString());
-            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
-            callback(null, tile, headers);
-        });
-    };
-
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
         Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);

lib/cartodb/server_options.js

@@ -1,11 +1,15 @@
 var   _          = require('underscore')
     , Step       = require('step')
-    , cartoData  = require('./carto_data');
+    , cartoData  = require('./carto_data')
+    , Cache = require('./cache_validator');
 
 module.exports = function(){
     var me = {
         base_url: '/tiles/:table',
-        grainstore: {datasource: global.environment.postgres},
+        grainstore: {
+          datasource: global.environment.postgres,
+          cachedir: global.environment.millstone.cache_basedir
+        },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
@@ -14,6 +18,28 @@ module.exports = function(){
         log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
     };
 
+    // Set the cache chanel info to invalidate the cache on the frontend server
+    //
+    // @param req The request object.
+    //            The function will have no effect unless req.res exists.
+    //            It is expected that req.params contains 'table' and 'dbname'
+    //
+    // @param cb function(err, channel) will be called when ready.
+    //           the channel parameter will be null if nothing was added
+    //
+    me.addCacheChannel = function(req, cb) {
+        // skip non-GET requests, or requests for which there's no response
+        if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
+        var res = req.res;
+        var ttl = global.environment.varnish.ttl || 86400;
+        Cache.generateCacheChannel(req, function(channel){
+            res.header('X-Cache-Channel', channel);
+            res.header('Last-Modified', new Date().toUTCString());
+            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
+            cb(null, channel); // add last-modified too ?
+        });
+    }
+
     /**
      * Whitelist input and get database name & default geometry type from
      * subdomain/user metadata held in CartoDB Redis
@@ -42,6 +68,8 @@ module.exports = function(){
           callback(null, xml);
         }
 
+        var that = this;
+
         Step(
             function getPrivacy(){
                 cartoData.authorize(req, this);
@@ -63,10 +91,14 @@ module.exports = function(){
                 cartoData.getGeometryType(req, this);
             },
             function finishSetup(err, data){
+                if ( err ) { callback(err, req); return; }
+
                 if (!_.isNull(data))
                     _.extend(req.params, {geom_type: data});
 
-                callback(err, req);
+                that.addCacheChannel(req, function(err, chan) {
+                  callback(err, req);
+                });
             }
         );
     };

package.json

@@ -22,12 +22,12 @@
         "cluster": "0.6.4",
         "node-varnish": "0.1.1",
         "underscore"  : "1.1.x",
-        "grainstore"  : "~0.5.0",
+        "grainstore"  : "~0.6.2",
-        "windshaft"  : "~0.4.13",
+        "windshaft"  : "~0.4.16",
         "step": "0.0.x",
         "generic-pool": "1.0.x",
-        "redis": "0.6.7",
+        "redis": "0.7.2",
-        "hiredis": "0.1.12",
+        "hiredis": "~0.1.12",
         "request": "2.9.202"
     },
     "devDependencies": {

test/acceptance/server.js

@@ -40,6 +40,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
+            headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' },
             body: '{"style":"#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}'
         }, function() { done(); });
     });
@@ -55,6 +56,8 @@ suite('server', function() {
         }, function(res) {
           // FIXME: should be 401 Unauthorized
           assert.equal(res.statusCode, 500, res.body);
+          assert.deepEqual(JSON.parse(res.body),
+            {error: 'Sorry, you are unauthorized (permission denied)'});
           done();
         });
     });
@@ -129,24 +132,33 @@ suite('server', function() {
     
     test("post'ing good style with no authentication returns an error", function(done){
         assert.response(server, {
-            url: '/tiles/my_table5/style',
+            url: '/tiles/my_table5/style?map_key=1234',
             method: 'POST',
             headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
-            data: querystring.stringify({style: 'Map {background-color:#aaa;}'})
+            data: querystring.stringify({style: 'Map {background-color:#fff;}'})
-        },{}, function(res) {
+        },{
-          // FIXME: should be 401 Unauthorized
+        }, function(res) {
-          assert.equal(res.statusCode, 500, res.body);
+          assert.equal(res.statusCode, 200, res.body);
-          assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
-
           assert.response(server, {
-              headers: {host: 'vizzuality.localhost.lan'},
               url: '/tiles/my_table5/style',
-              method: 'GET'
+              method: 'POST',
-          },{
+              headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
-              status: 200,
+              data: querystring.stringify({style: 'Map {background-color:#aaa;}'})
-              body: JSON.stringify({style: 'Map {background-color:#fff;}'})
+          },{}, function(res) {
-          }, function() { done(); });
+            // FIXME: should be 401 Unauthorized
+            assert.equal(res.statusCode, 500, res.body);
+            assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
 
+            assert.response(server, {
+                headers: {host: 'vizzuality.localhost.lan'},
+                url: '/tiles/my_table5/style',
+                method: 'GET'
+            },{
+                status: 200,
+                body: JSON.stringify({style: 'Map {background-color:#fff;}'})
+            }, function() { done(); });
+
+          });
         });
     });
 
@@ -216,7 +228,7 @@ suite('server', function() {
 
             // Retrive style with authenticated request
             assert.response(server, {
-                headers: {host: 'localhost'},
+                headers: {host: 'vizzuality'},
                 url: '/tiles/my_table5/style?map_key=1234',
                 method: 'GET'
             },{}, function(res) {
@@ -225,7 +237,7 @@ suite('server', function() {
 
               // Now retrive style with unauthenticated request
               assert.response(server, {
-                  headers: {host: 'localhost'},
+                  headers: {host: 'vizzuality'},
                   url: '/tiles/my_table5/style',
                   method: 'GET'
               }, {}, function(res) {
@@ -252,6 +264,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
+            headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_tablez' },
             body: '{"infowindow":null}'
         }, function() { done(); });
     });
@@ -317,7 +330,8 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' }
+            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8',
+                       'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' }
         }, function() { done(); });
     });
     
@@ -382,7 +396,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'image/png' }
+            headers: { 'Content-Type': 'image/png', 'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' }
         }, function() { done(); });
     });
     

tools/reset_styles

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+
+// Reset redis-stored XML styles so that they are regenerated
+// from CartoCSS on next tile request
+
+var redis = require('redis')
+
+var REDIS_PORT = 6379; // TODO: make a parameter
+
+var dbnum = 0;
+
+var client = redis.createClient(REDIS_PORT, 'localhost');
+client.on('connect', function() {
+  client.select(dbnum);
+  client.keys('map_style|*', function(err, matches) {
+
+    processNext = function() {
+      if ( ! matches.length ) process.exit(0);
+      var k = matches.shift();
+      console.log("Resetting XML in key: " + k);
+      client.get(k, function(err, val) {
+        if ( err ) throw err;
+        val = JSON.parse(val);
+        delete val.xml;
+        client.set(k, JSON.stringify(val), function() {
+          console.log("done with style " + k);
+          processNext();
+        });
+      });
+    }
+
+    processNext();
+
+  });
+});