cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enable cache clearing at table level granularity

Browse Source
This commit is contained in:
Simon Tokumine 2012-05-02 19:32:54 +01:00
parent d5759db8ca
commit 4078098c3f
8 changed files with 219 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/environments/development.js
View File

@ -22,8 +22,10 @@ var config = {
reapIntervalMillis: 1 reapIntervalMillis: 1
} }
,sqlapi: { ,sqlapi: {
host: '127.0.0.1', protocol: 'http',
port: 8080 host: 'localhost.lan',
port: 8080,
version: 'v1'
} }
,varnish: { ,varnish: {
host: 'localhost', host: 'localhost',

8
config/environments/production.js
View File

@ -1,5 +1,5 @@
var config = { var config = {
environment: 'production' environment: 'production'
,port: 8181 ,port: 8181
,host: '127.0.0.1' ,host: '127.0.0.1'
,enable_cors: true ,enable_cors: true
@ -15,8 +15,10 @@ var config = {
port: 6379 port: 6379
} }
,sqlapi: { ,sqlapi: {
host: '127.0.0.1', protocol: 'https',
port: 8080 host: 'cartodb.com',
port: 8080,
version: 'v2'
} }
,varnish: { ,varnish: {
host: 'localhost', host: 'localhost',

6
config/environments/test.js
View File

@ -19,8 +19,10 @@ var config = {
reapIntervalMillis: 1 reapIntervalMillis: 1
} }
,sqlapi: { ,sqlapi: {
host: '127.0.0.1', protocol: 'http',
port: 8080 host: 'localhost.lan',
port: 8080,
version: 'v1'
} }
,varnish: { ,varnish: {
host: '', host: '',

85
lib/cartodb/cache_validator.js
View File

@ -1,17 +1,88 @@
var _ = require('underscore'), var _ = require('underscore'),
Varnish = require('node-varnish'); Varnish = require('node-varnish'),
request = require('request'),
var varnish_queue = null; crypto = require('crypto'),
channelCache = {},
varnish_queue = null;
function init(host, port) { function init(host, port) {
varnish_queue = new Varnish.VarnishQueue(host, port); varnish_queue = new Varnish.VarnishQueue(host, port);
} }
function invalidate_db(dbname) { function invalidate_db(dbname, table) {
varnish_queue.run_cmd('purge obj.http.X-Cache-Channel == ' + dbname); try{
varnish_queue.run_cmd('purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
console.log('[SUCCESS FLUSHING CACHE]');
} catch (e) {
console.log("[ERROR FLUSHING CACHE] Is enable_cache set to true? Failed for: " + 'purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
}
}
function generateCacheChannel(req, callback){
var cacheChannel = "";
// use key to call sql api with sql request if present, else just return dbname and table name
// base key
var tableNames = req.params.table;
var dbName = req.params.dbname;
var username = req.headers.host.split('.')[0];
// replace tableNames with the results of the explain if present
if (_.isString(req.params.sql) && req.params.sql != ''){
// initialise MD5 key of sql for cache lookups
var sql_md5 = generateMD5(req.params.sql);
var api = global.environment.sqlapi;
var qs = {};
// use cache if present
if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
callback(channelCache[sql_md5]);
} else{
// strip out windshaft/mapnik inserted sql
var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/)[1];
// build up api string
var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
// add query to querystring
qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// add api_key if present in tile request (means table is private)
if (_.isString(req.params.map_key) && req.params.map_key != ''){
qs.api_key = req.params.map_key;
}
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){
if (!err && response.statusCode == 200) {
tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
} else {
//oops, no SQL API. Just cache using fallback 'table' key
tableNames = 'table';
}
cacheChannel = buildCacheChannel(dbName,tableNames);
channelCache[sql_md5] = cacheChannel; // store for caching
callback(cacheChannel);
});
}
} else {
cacheChannel = buildCacheChannel(dbName,tableNames);
callback(cacheChannel);
}
}
function buildCacheChannel(dbName, tableNames){
return dbName + ':' + tableNames;
}
function generateMD5(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
} }
module.exports = { module.exports = {
init: init, init: init,
invalidate_db: invalidate_db invalidate_db: invalidate_db,
generateCacheChannel: generateCacheChannel
} }

7
lib/cartodb/carto_data.js
View File

@ -18,13 +18,12 @@ module.exports = function() {
var me = { var me = {
user_metadata_db: 5, user_metadata_db: 5,
table_metadata_db: 0, table_metadata_db: 0,
user_key: "rails:users:<%= username %>", user_key: "rails:users:<%= username %>",
map_key: "rails:users:<%= username %>:map_key", map_key: "rails:users:<%= username %>:map_key",
table_key: "rails:<%= database_name %>:<%= table_name %>" table_key: "rails:<%= database_name %>:<%= table_name %>"
}; };
/** /**
* Get the database name for this particular subdomain/username * Get the database name for this particular subdomain/username
* *
@ -56,7 +55,7 @@ module.exports = function() {
}; };
/** /**
* Get the user map key for this particular subdomain/username * Check the user map key for this particular subdomain/username
* *
* @param req - standard express req object. importantly contains host information * @param req - standard express req object. importantly contains host information
* @param callback * @param callback

30
lib/cartodb/cartodb_windshaft.js
View File

@ -8,17 +8,19 @@ var CartodbWindshaft = function(serverOptions) {
// set the cache chanel info to invalidate the cache on the frontend server // set the cache chanel info to invalidate the cache on the frontend server
serverOptions.afterTileRender = function(req, res, tile, headers, callback) { serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
res.header('X-Cache-Channel', req.params.dbname); Cache.generateCacheChannel(req, function(channel){
res.header('Last-Modified', new Date().toUTCString()); res.header('X-Cache-Channel', channel);
res.header('Cache-Control', 'no-cache,max-age=86400,must-revalidate, public'); res.header('Last-Modified', new Date().toUTCString());
callback(null, tile, headers); res.header('Cache-Control', 'no-cache,max-age=86400,must-revalidate, public');
callback(null, tile, headers);
});
}; };
if(serverOptions.cache_enabled) { if(serverOptions.cache_enabled) {
console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
Cache.init(serverOptions.varnish_host, serverOptions.varnish_port); Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
serverOptions.afterStateChange = function(req, data, callback) { serverOptions.afterStateChange = function(req, data, callback) {
Cache.invalidate_db(req.params.dbname); Cache.invalidate_db(req.params.dbname, req.params.table);
callback(null, data); callback(null, data);
} }
} }
@ -65,6 +67,24 @@ var CartodbWindshaft = function(serverOptions) {
); );
}); });
/**
* Helper API to allow per table tile cache (and sql cache) to be invalidated remotely.
* TODO: Move?
*/
ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
Step(
function(){
serverOptions.flushCache(req, Cache, this);
},
function(err, data){
if (err){
res.send(500);
} else {
res.send({status: 'ok'}, 200);
}
}
);
});
return ws; return ws;
} }

175
lib/cartodb/server_options.js
View File

@ -10,96 +10,117 @@ module.exports = function(){
enable_cors: global.environment.enable_cors, enable_cors: global.environment.enable_cors,
varnish_host: global.environment.varnish.host, varnish_host: global.environment.varnish.host,
varnish_port: global.environment.varnish.port, varnish_port: global.environment.varnish.port,
cache_enabled: global.environment.cache_enabled,
log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m' log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
}; };
/** /**
* Whitelist input and get database name & default geometry type from * Whitelist input and get database name & default geometry type from
* subdomain/user metadata held in CartoDB Redis * subdomain/user metadata held in CartoDB Redis
* @param req - standard express request obj. Should have host & table * @param req - standard express request obj. Should have host & table
* @param callback * @param callback
*/ */
me.req2params = function(req, callback){ me.req2params = function(req, callback){
// Whitelist query parameters and attach format // Whitelist query parameters and attach format
var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style']; var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style'];
var bad_query = _.difference(_.keys(req.query), good_query); var bad_query = _.difference(_.keys(req.query), good_query);
_.each(bad_query, function(key){ delete req.query[key]; }); _.each(bad_query, function(key){ delete req.query[key]; });
req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
// bring all query values onto req.params object // bring all query values onto req.params object
_.extend(req.params, req.query); _.extend(req.params, req.query);
// for cartodb, ensure interactivity is cartodb_id or user specified // for cartodb, ensure interactivity is cartodb_id or user specified
req.params.interactivity = req.params.interactivity || 'cartodb_id'; req.params.interactivity = req.params.interactivity || 'cartodb_id';
Step( Step(
function getPrivacy(){ function getPrivacy(){
cartoData.authorize(req, this); cartoData.authorize(req, this);
}, },
function gatekeep(err, data){ function gatekeep(err, data){
if(err) throw err; if(err) throw err;
if(data === "0") throw new Error("Sorry, you are unauthorized"); if(data === "0") throw new Error("Sorry, you are unauthorized");
return data; return data;
}, },
function getDatabase(err, data){ function getDatabase(err, data){
if(err) throw err; if(err) throw err;
cartoData.getDatabase(req, this); cartoData.getDatabase(req, this);
}, },
function getGeometryType(err, data){ function getGeometryType(err, data){
if (err) throw err; if (err) throw err;
_.extend(req.params, {dbname:data}); _.extend(req.params, {dbname:data});
cartoData.getGeometryType(req, this); cartoData.getGeometryType(req, this);
}, },
function finishSetup(err, data){ function finishSetup(err, data){
if (!_.isNull(data)) if (!_.isNull(data))
_.extend(req.params, {geom_type: data}); _.extend(req.params, {geom_type: data});
callback(err, req); callback(err, req);
} }
); );
}; };
/** /**
* Little helper method to get the current list of infowindow variables and return to client * Little helper method to get the current list of infowindow variables and return to client
* @param req * @param req
* @param callback * @param callback
*/ */
me.getInfowindow = function(req, callback){ me.getInfowindow = function(req, callback){
var that = this; var that = this;
Step( Step(
function(){ function(){
that.req2params(req, this); that.req2params(req, this);
}, },
function(err, data){ function(err, data){
if (err) throw err; if (err) throw err;
cartoData.getInfowindow(data, callback); cartoData.getInfowindow(data, callback);
} }
); );
}; };
/** /**
* Little helper method to get map metadata and return to client * Little helper method to get map metadata and return to client
* @param req * @param req
* @param callback * @param callback
*/ */
me.getMapMetadata = function(req, callback){ me.getMapMetadata = function(req, callback){
var that = this; var that = this;
Step( Step(
function(){ function(){
that.req2params(req, this); that.req2params(req, this);
}, },
function(err, data){ function(err, data){
if (err) throw err; if (err) throw err;
cartoData.getMapMetadata(data, callback); cartoData.getMapMetadata(data, callback);
} }
); );
}; };
return me; /**
* Helper to clear out tile cache on request
* @param req
* @param callback
*/
me.flushCache = function(req, Cache, callback){
var that = this;
Step(
function(){
that.req2params(req, this);
},
function(err, data){
if (err) throw err;
Cache.invalidate_db(req.params.dbname, req.params.table);
callback(null, true);
}
);
};
return me;
}(); }();

3
package.json
View File

@ -22,7 +22,8 @@
"step": "0.0.x", "step": "0.0.x",
"generic-pool": "1.0.x", "generic-pool": "1.0.x",
"redis": "0.6.7", "redis": "0.6.7",
"hiredis": "0.1.12" "hiredis": "0.1.12",
"request": "2.9.202"
}, },
"devDependencies": { "devDependencies": {
"expresso": "0.8.x" "expresso": "0.8.x"