cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix X-Cache-Channel for multilayer (by token) responses

Browse Source 
Required upgrading Windshaft to 0.9.2
Includes testcases
This commit is contained in:
Sandro Santilli 2013-03-13 16:45:15 +01:00
parent 899d8a3c64
commit dfc4a02398
6 changed files with 147 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -1,6 +1,7 @@
1.1.9
-----
* Handle SQL API errors by requesting no Varnish cache
* Fix X-Cache-Channel for multilayer (by token) responses
1.1.8
-----

77
lib/cartodb/cache_validator.js
View File

@ -1,8 +1,5 @@
var _ = require('underscore'),
Varnish = require('node-varnish'),
request = require('request'),
crypto = require('crypto'),
channelCache = {},
varnish_queue = null;
function init(host, port) {
@ -18,79 +15,7 @@ function invalidate_db(dbname, table) {
}
}
function generateCacheChannel(req, callback){
var cacheChannel = "";
// use key to call sql api with sql request if present, else just return dbname and table name
// base key
var tableNames = req.params.table;
var dbName = req.params.dbname;
var username = req.headers.host.split('.')[0];
// replace tableNames with the results of the explain if present
if (_.isString(req.params.sql) && req.params.sql != ''){
// initialise MD5 key of sql for cache lookups
var sql_md5 = generateMD5(req.params.sql);
var api = global.environment.sqlapi;
var qs = {};
// use cache if present
if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
callback(null, channelCache[sql_md5]);
} else{
// strip out windshaft/mapnik inserted sql if present
var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
sql = (sql != null) ? sql[1] : req.params.sql;
// build up api string
var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
// add query to querystring
qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// add api_key if present in tile request (means table is private)
if (_.isString(req.params.map_key) && req.params.map_key != ''){
qs.api_key = req.params.map_key;
}
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
var epref = 'could not detect source tables using SQL api at ' + sqlapi;
if (err){
var msg = err.message ? err.message : err;
callback(new Error(epref + ': ' + msg));
return;
}
if (res.statusCode != 200) {
var msg = res.body.error ? res.body.error : res.body;
callback(new Error(epref + ': ' + msg));
return;
}
var qtables = body.rows[0].cdb_querytables;
tableNames = qtables.split(/^\{(.*)\}$/)[1];
cacheChannel = buildCacheChannel(dbName,tableNames);
channelCache[sql_md5] = cacheChannel; // store for caching
callback(null, cacheChannel);
});
}
} else {
cacheChannel = buildCacheChannel(dbName,tableNames);
callback(null, cacheChannel);
}
}
function buildCacheChannel(dbName, tableNames){
return dbName + ':' + tableNames;
}
function generateMD5(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
module.exports = {
init: init,
invalidate_db: invalidate_db,
generateCacheChannel: generateCacheChannel
invalidate_db: invalidate_db
}

139
lib/cartodb/server_options.js
View File

@ -3,6 +3,8 @@ var _ = require('underscore')
, cartoData = require('./carto_data')
, Cache = require('./cache_validator')
, mapnik = require('mapnik')
, crypto = require('crypto')
, request = require('request')
;
module.exports = function(){
@ -44,6 +46,111 @@ module.exports = function(){
+ me.grainstore.mapnik_version + ")");
}
/* This whole block is about generating X-Cache-Channel { */
// TODO: review lifetime of elements of this cache
// NOTE: by-token indices should only be dropped when
// the corresponding layegroup is dropped, because
// we have no SQL after layer creation.
me.channelCache = {};
me.affectedTables = function (username, api_key, sql, callback) {
var api = global.environment.sqlapi;
// build up api string
var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
var qs = {};
// add query to querystring
qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// add api_key if given
if (_.isString(api_key) && api_key != ''){ qs.api_key = api_key; }
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
var epref = 'could not detect source tables using SQL api at ' + sqlapi;
if (err){
var msg = err.message ? err.message : err;
callback(new Error(epref + ': ' + msg));
return;
}
if (res.statusCode != 200) {
var msg = res.body.error ? res.body.error : res.body;
callback(new Error(epref + ': ' + msg));
return;
}
var qtables = body.rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
callback(null, tableNames);
});
},
me.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames;
};
me.generateMD5 = function(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
me.generateCacheChannel = function(req, callback){
// use key to call sql api with sql request if present, else
// just return dbname and table name base key
var dbName = req.params.dbname;
var cacheKey = [ dbName ];
if ( req.params.token ) cacheKey.push(req.params.token);
else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
cacheKey = cacheKey.join(':');
if ( me.channelCache.hasOwnProperty(cacheKey) ) {
callback(null, me.channelCache[cacheKey]);
return;
}
if ( req.params.token ) {
if ( ! me.channelCache.hasOwnProperty(cacheKey) ) {
callback(new Error('missing channel cache for token ' + req.params.token));
} else {
callback(null, me.channelCache[cacheKey]);
}
return;
}
if ( ! req.params.sql && ! req.params.token ) {
var cacheChannel = me.buildCacheChannel(dbName, req.params.table);
// not worth caching this
callback(null, cacheChannel);
return;
}
if ( ! req.params.sql ) {
console.log('req:'); console.dir(req);
callback(new Error("this request doesn't need an X-Cache-Channel generated"));
return;
}
var dbName = req.params.dbname;
var username = req.headers.host.split('.')[0];
// strip out windshaft/mapnik inserted sql if present
var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
sql = (sql != null) ? sql[1] : req.params.sql;
me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) {
if ( err ) { callback(err); return; }
var cacheChannel = me.buildCacheChannel(dbName,tableNames);
me.channelCache[cacheKey] = cacheChannel; // store for caching
callback(null, cacheChannel);
});
};
// Set the cache chanel info to invalidate the cache on the frontend server
//
// @param req The request object.
@ -65,7 +172,8 @@ module.exports = function(){
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
}
Cache.generateCacheChannel(req, function(err, channel){
me.generateCacheChannel(req, function(err, channel){
if ( ! err ) {
res.header('X-Cache-Channel', channel);
cb(null, channel);
@ -75,7 +183,34 @@ module.exports = function(){
cb(null, 'ERROR');
}
});
}
};
me.afterLayergroupCreate = function(req, response, callback) {
var token = response.layergroupid;
var mapconfig = req.body;
var sql = [];
_.each(mapconfig.layers, function(lyr) {
sql.push(lyr.options.sql);
});
sql = sql.join(';');
console.log('afterLayergroupCreate: sql:'+sql);
var dbName = req.params.dbname;
var usr = req.headers.host.split('.')[0];
var key = req.params.map_key;
var cacheKey = dbName + ':' + token;
me.affectedTables(usr, key, sql, function(err, tableNames) {
if ( err ) { callback(err); return; }
var cacheChannel = me.buildCacheChannel(dbName,tableNames);
me.channelCache[cacheKey] = cacheChannel; // store for caching
callback(null);
});
};
/* X-Cache-Channel generation } */
/**
* Whitelist input and get database name & default geometry type from

4
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.1.7",
"version": "1.1.9",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb02",
@ -257,7 +257,7 @@
}
},
"windshaft": {
"version": "0.9.1",
"version": "0.9.2",
"dependencies": {
"express": {
"version": "2.5.11",

2
package.json
View File

@ -22,7 +22,7 @@
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"grainstore" : "~0.11.1",
"windshaft" : "~0.9.1",
"windshaft" : "~0.9.2",
"step": "0.0.x",
"generic-pool": "~1.0.12",
"redis": "0.7.2",

13
test/acceptance/multilayer.js
View File

@ -92,18 +92,15 @@ suite('multilayer', function() {
// Check X-Cache-Channel
var cc = res.headers['x-cache-channel'];
assert.ok(cc);
assert.ok(cc);
var dbname = 'cartodb_test_user_1_db'
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
//console.log('jsonquery: '+ jsonquery);
// FIXME: this is currently _undefined_ !
/*
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.api_key, qo.map_key);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)');
*/
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql + ';'
+ layergroup.layers[1].options.sql
+ '$windshaft$)');
assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2,
function(err, similarity) {