From dfc4a0239830346305b861e190593f8f9b63d55d Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 13 Mar 2013 16:45:15 +0100 Subject: [PATCH] Fix X-Cache-Channel for multilayer (by token) responses Required upgrading Windshaft to 0.9.2 Includes testcases --- NEWS.md | 1 + lib/cartodb/cache_validator.js | 77 +----------------- lib/cartodb/server_options.js | 139 ++++++++++++++++++++++++++++++++- npm-shrinkwrap.json | 4 +- package.json | 2 +- test/acceptance/multilayer.js | 13 ++- 6 files changed, 147 insertions(+), 89 deletions(-) diff --git a/NEWS.md b/NEWS.md index 51b83821..14bd722c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ 1.1.9 ----- * Handle SQL API errors by requesting no Varnish cache +* Fix X-Cache-Channel for multilayer (by token) responses 1.1.8 ----- diff --git a/lib/cartodb/cache_validator.js b/lib/cartodb/cache_validator.js index aa551df6..8449861e 100644 --- a/lib/cartodb/cache_validator.js +++ b/lib/cartodb/cache_validator.js @@ -1,8 +1,5 @@ var _ = require('underscore'), Varnish = require('node-varnish'), - request = require('request'), - crypto = require('crypto'), - channelCache = {}, varnish_queue = null; function init(host, port) { @@ -18,79 +15,7 @@ function invalidate_db(dbname, table) { } } -function generateCacheChannel(req, callback){ - var cacheChannel = ""; - - // use key to call sql api with sql request if present, else just return dbname and table name - // base key - var tableNames = req.params.table; - var dbName = req.params.dbname; - var username = req.headers.host.split('.')[0]; - - // replace tableNames with the results of the explain if present - if (_.isString(req.params.sql) && req.params.sql != ''){ - // initialise MD5 key of sql for cache lookups - var sql_md5 = generateMD5(req.params.sql); - var api = global.environment.sqlapi; - var qs = {}; - - // use cache if present - if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) { - callback(null, channelCache[sql_md5]); - } else{ - // strip out windshaft/mapnik inserted sql if present - var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/); - sql = (sql != null) ? sql[1] : req.params.sql; - - // build up api string - var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql' - - // add query to querystring - qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; - - // add api_key if present in tile request (means table is private) - if (_.isString(req.params.map_key) && req.params.map_key != ''){ - qs.api_key = req.params.map_key; - } - - // call sql api - request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){ - var epref = 'could not detect source tables using SQL api at ' + sqlapi; - if (err){ - var msg = err.message ? err.message : err; - callback(new Error(epref + ': ' + msg)); - return; - } - if (res.statusCode != 200) { - var msg = res.body.error ? res.body.error : res.body; - callback(new Error(epref + ': ' + msg)); - return; - } - var qtables = body.rows[0].cdb_querytables; - tableNames = qtables.split(/^\{(.*)\}$/)[1]; - cacheChannel = buildCacheChannel(dbName,tableNames); - channelCache[sql_md5] = cacheChannel; // store for caching - callback(null, cacheChannel); - }); - } - } else { - cacheChannel = buildCacheChannel(dbName,tableNames); - callback(null, cacheChannel); - } -} - -function buildCacheChannel(dbName, tableNames){ - return dbName + ':' + tableNames; -} - -function generateMD5(data){ - var hash = crypto.createHash('md5'); - hash.update(data); - return hash.digest('hex'); -} - module.exports = { init: init, - invalidate_db: invalidate_db, - generateCacheChannel: generateCacheChannel + invalidate_db: invalidate_db } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index ef847306..328b2e87 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -3,6 +3,8 @@ var _ = require('underscore') , cartoData = require('./carto_data') , Cache = require('./cache_validator') , mapnik = require('mapnik') + , crypto = require('crypto') + , request = require('request') ; module.exports = function(){ @@ -44,6 +46,111 @@ module.exports = function(){ + me.grainstore.mapnik_version + ")"); } +/* This whole block is about generating X-Cache-Channel { */ + + // TODO: review lifetime of elements of this cache + // NOTE: by-token indices should only be dropped when + // the corresponding layegroup is dropped, because + // we have no SQL after layer creation. + me.channelCache = {}; + + me.affectedTables = function (username, api_key, sql, callback) { + + var api = global.environment.sqlapi; + + // build up api string + var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql' + + var qs = {}; + + // add query to querystring + qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; + + // add api_key if given + if (_.isString(api_key) && api_key != ''){ qs.api_key = api_key; } + + // call sql api + request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){ + var epref = 'could not detect source tables using SQL api at ' + sqlapi; + if (err){ + var msg = err.message ? err.message : err; + callback(new Error(epref + ': ' + msg)); + return; + } + if (res.statusCode != 200) { + var msg = res.body.error ? res.body.error : res.body; + callback(new Error(epref + ': ' + msg)); + return; + } + var qtables = body.rows[0].cdb_querytables; + var tableNames = qtables.split(/^\{(.*)\}$/)[1]; + callback(null, tableNames); + }); + }, + + me.buildCacheChannel = function (dbName, tableNames){ + return dbName + ':' + tableNames; + }; + + me.generateMD5 = function(data){ + var hash = crypto.createHash('md5'); + hash.update(data); + return hash.digest('hex'); + } + + me.generateCacheChannel = function(req, callback){ + + // use key to call sql api with sql request if present, else + // just return dbname and table name base key + var dbName = req.params.dbname; + + var cacheKey = [ dbName ]; + if ( req.params.token ) cacheKey.push(req.params.token); + else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) ); + cacheKey = cacheKey.join(':'); + + if ( me.channelCache.hasOwnProperty(cacheKey) ) { + callback(null, me.channelCache[cacheKey]); + return; + } + + if ( req.params.token ) { + if ( ! me.channelCache.hasOwnProperty(cacheKey) ) { + callback(new Error('missing channel cache for token ' + req.params.token)); + } else { + callback(null, me.channelCache[cacheKey]); + } + return; + } + + if ( ! req.params.sql && ! req.params.token ) { + var cacheChannel = me.buildCacheChannel(dbName, req.params.table); + // not worth caching this + callback(null, cacheChannel); + return; + } + + if ( ! req.params.sql ) { +console.log('req:'); console.dir(req); + callback(new Error("this request doesn't need an X-Cache-Channel generated")); + return; + } + + var dbName = req.params.dbname; + var username = req.headers.host.split('.')[0]; + + // strip out windshaft/mapnik inserted sql if present + var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/); + sql = (sql != null) ? sql[1] : req.params.sql; + + me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) { + if ( err ) { callback(err); return; } + var cacheChannel = me.buildCacheChannel(dbName,tableNames); + me.channelCache[cacheKey] = cacheChannel; // store for caching + callback(null, cacheChannel); + }); + }; + // Set the cache chanel info to invalidate the cache on the frontend server // // @param req The request object. @@ -65,7 +172,8 @@ module.exports = function(){ res.header('Last-Modified', new Date().toUTCString()); res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); } - Cache.generateCacheChannel(req, function(err, channel){ + + me.generateCacheChannel(req, function(err, channel){ if ( ! err ) { res.header('X-Cache-Channel', channel); cb(null, channel); @@ -75,7 +183,34 @@ module.exports = function(){ cb(null, 'ERROR'); } }); - } + }; + + me.afterLayergroupCreate = function(req, response, callback) { + var token = response.layergroupid; + var mapconfig = req.body; + + var sql = []; + _.each(mapconfig.layers, function(lyr) { + sql.push(lyr.options.sql); + }); + sql = sql.join(';'); + console.log('afterLayergroupCreate: sql:'+sql); + + var dbName = req.params.dbname; + var usr = req.headers.host.split('.')[0]; + var key = req.params.map_key; + + var cacheKey = dbName + ':' + token; + + me.affectedTables(usr, key, sql, function(err, tableNames) { + if ( err ) { callback(err); return; } + var cacheChannel = me.buildCacheChannel(dbName,tableNames); + me.channelCache[cacheKey] = cacheChannel; // store for caching + callback(null); + }); + }; + +/* X-Cache-Channel generation } */ /** * Whitelist input and get database name & default geometry type from diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 3ef03a44..b61b56fe 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "1.1.7", + "version": "1.1.9", "dependencies": { "cluster2": { "version": "0.3.5-cdb02", @@ -257,7 +257,7 @@ } }, "windshaft": { - "version": "0.9.1", + "version": "0.9.2", "dependencies": { "express": { "version": "2.5.11", diff --git a/package.json b/package.json index d28ef1fb..cad96acd 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "node-varnish": "0.1.1", "underscore" : "~1.3.3", "grainstore" : "~0.11.1", - "windshaft" : "~0.9.1", + "windshaft" : "~0.9.2", "step": "0.0.x", "generic-pool": "~1.0.12", "redis": "0.7.2", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index ab884dc6..2b1a27dd 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -92,18 +92,15 @@ suite('multilayer', function() { // Check X-Cache-Channel var cc = res.headers['x-cache-channel']; - assert.ok(cc); + assert.ok(cc); var dbname = 'cartodb_test_user_1_db' assert.equal(cc.substring(0, dbname.length), dbname); var jsonquery = cc.substring(dbname.length+1); -//console.log('jsonquery: '+ jsonquery); - // FIXME: this is currently _undefined_ ! - -/* var sentquery = JSON.parse(jsonquery); - assert.equal(sentquery.api_key, qo.map_key); - assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)'); -*/ + assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + ';' + + layergroup.layers[1].options.sql + + '$windshaft$)'); assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2, function(err, similarity) {