cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enable cache clearing at table level granularity

Browse Source
This commit is contained in:
Simon Tokumine 2012-05-02 19:32:54 +01:00
parent d5759db8ca
commit 4078098c3f
8 changed files with 219 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/environments/development.js
View File

@ -22,8 +22,10 @@ var config = {
reapIntervalMillis: 1
}
,sqlapi: {
host: '127.0.0.1',
port: 8080
protocol: 'http',
host: 'localhost.lan',
port: 8080,
version: 'v1'
}
,varnish: {
host: 'localhost',

8
config/environments/production.js
View File

@ -1,5 +1,5 @@
var config = {
environment: 'production'
environment: 'production'
,port: 8181
,host: '127.0.0.1'
,enable_cors: true
@ -15,8 +15,10 @@ var config = {
port: 6379
}
,sqlapi: {
host: '127.0.0.1',
port: 8080
protocol: 'https',
host: 'cartodb.com',
port: 8080,
version: 'v2'
}
,varnish: {
host: 'localhost',

6
config/environments/test.js
View File

@ -19,8 +19,10 @@ var config = {
reapIntervalMillis: 1
}
,sqlapi: {
host: '127.0.0.1',
port: 8080
protocol: 'http',
host: 'localhost.lan',
port: 8080,
version: 'v1'
}
,varnish: {
host: '',

85
lib/cartodb/cache_validator.js
View File

@ -1,17 +1,88 @@
var _ = require('underscore'),
Varnish = require('node-varnish');
var varnish_queue = null;
var _ = require('underscore'),
Varnish = require('node-varnish'),
request = require('request'),
crypto = require('crypto'),
channelCache = {},
varnish_queue = null;
function init(host, port) {
varnish_queue = new Varnish.VarnishQueue(host, port);
}
function invalidate_db(dbname) {
varnish_queue.run_cmd('purge obj.http.X-Cache-Channel == ' + dbname);
function invalidate_db(dbname, table) {
try{
varnish_queue.run_cmd('purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
console.log('[SUCCESS FLUSHING CACHE]');
} catch (e) {
console.log("[ERROR FLUSHING CACHE] Is enable_cache set to true? Failed for: " + 'purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
}
}
function generateCacheChannel(req, callback){
var cacheChannel = "";
// use key to call sql api with sql request if present, else just return dbname and table name
// base key
var tableNames = req.params.table;
var dbName = req.params.dbname;
var username = req.headers.host.split('.')[0];
// replace tableNames with the results of the explain if present
if (_.isString(req.params.sql) && req.params.sql != ''){
// initialise MD5 key of sql for cache lookups
var sql_md5 = generateMD5(req.params.sql);
var api = global.environment.sqlapi;
var qs = {};
// use cache if present
if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
callback(channelCache[sql_md5]);
} else{
// strip out windshaft/mapnik inserted sql
var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/)[1];
// build up api string
var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
// add query to querystring
qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// add api_key if present in tile request (means table is private)
if (_.isString(req.params.map_key) && req.params.map_key != ''){
qs.api_key = req.params.map_key;
}
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){
if (!err && response.statusCode == 200) {
tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
} else {
//oops, no SQL API. Just cache using fallback 'table' key
tableNames = 'table';
}
cacheChannel = buildCacheChannel(dbName,tableNames);
channelCache[sql_md5] = cacheChannel; // store for caching
callback(cacheChannel);
});
}
} else {
cacheChannel = buildCacheChannel(dbName,tableNames);
callback(cacheChannel);
}
}
function buildCacheChannel(dbName, tableNames){
return dbName + ':' + tableNames;
}
function generateMD5(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
module.exports = {
init: init,
invalidate_db: invalidate_db
invalidate_db: invalidate_db,
generateCacheChannel: generateCacheChannel
}

7
lib/cartodb/carto_data.js
View File

@ -18,13 +18,12 @@ module.exports = function() {
var me = {
user_metadata_db: 5,
table_metadata_db: 0,
user_key: "rails:users:<%= username %>",
map_key: "rails:users:<%= username %>:map_key",
user_key: "rails:users:<%= username %>",
map_key: "rails:users:<%= username %>:map_key",
table_key: "rails:<%= database_name %>:<%= table_name %>"
};
/**
* Get the database name for this particular subdomain/username
*
@ -56,7 +55,7 @@ module.exports = function() {
};
/**
* Get the user map key for this particular subdomain/username
* Check the user map key for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback

30
lib/cartodb/cartodb_windshaft.js
View File

@ -8,17 +8,19 @@ var CartodbWindshaft = function(serverOptions) {
// set the cache chanel info to invalidate the cache on the frontend server
serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
res.header('X-Cache-Channel', req.params.dbname);
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age=86400,must-revalidate, public');
callback(null, tile, headers);
Cache.generateCacheChannel(req, function(channel){
res.header('X-Cache-Channel', channel);
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age=86400,must-revalidate, public');
callback(null, tile, headers);
});
};
if(serverOptions.cache_enabled) {
console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
serverOptions.afterStateChange = function(req, data, callback) {
Cache.invalidate_db(req.params.dbname);
Cache.invalidate_db(req.params.dbname, req.params.table);
callback(null, data);
}
}
@ -65,6 +67,24 @@ var CartodbWindshaft = function(serverOptions) {
);
});
/**
* Helper API to allow per table tile cache (and sql cache) to be invalidated remotely.
* TODO: Move?
*/
ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
Step(
function(){
serverOptions.flushCache(req, Cache, this);
},
function(err, data){
if (err){
res.send(500);
} else {
res.send({status: 'ok'}, 200);
}
}
);
});
return ws;
}

175
lib/cartodb/server_options.js
View File

@ -10,96 +10,117 @@ module.exports = function(){
enable_cors: global.environment.enable_cors,
varnish_host: global.environment.varnish.host,
varnish_port: global.environment.varnish.port,
cache_enabled: global.environment.cache_enabled,
log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
};
};
/**
* Whitelist input and get database name & default geometry type from
* subdomain/user metadata held in CartoDB Redis
* @param req - standard express request obj. Should have host & table
* @param callback
*/
me.req2params = function(req, callback){
/**
* Whitelist input and get database name & default geometry type from
* subdomain/user metadata held in CartoDB Redis
* @param req - standard express request obj. Should have host & table
* @param callback
*/
me.req2params = function(req, callback){
// Whitelist query parameters and attach format
var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style'];
var bad_query = _.difference(_.keys(req.query), good_query);
// Whitelist query parameters and attach format
var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style'];
var bad_query = _.difference(_.keys(req.query), good_query);
_.each(bad_query, function(key){ delete req.query[key]; });
req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
_.each(bad_query, function(key){ delete req.query[key]; });
req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
// bring all query values onto req.params object
_.extend(req.params, req.query);
// bring all query values onto req.params object
_.extend(req.params, req.query);
// for cartodb, ensure interactivity is cartodb_id or user specified
req.params.interactivity = req.params.interactivity || 'cartodb_id';
// for cartodb, ensure interactivity is cartodb_id or user specified
req.params.interactivity = req.params.interactivity || 'cartodb_id';
Step(
function getPrivacy(){
cartoData.authorize(req, this);
},
function gatekeep(err, data){
if(err) throw err;
if(data === "0") throw new Error("Sorry, you are unauthorized");
return data;
},
function getDatabase(err, data){
if(err) throw err;
Step(
function getPrivacy(){
cartoData.authorize(req, this);
},
function gatekeep(err, data){
if(err) throw err;
if(data === "0") throw new Error("Sorry, you are unauthorized");
return data;
},
function getDatabase(err, data){
if(err) throw err;
cartoData.getDatabase(req, this);
},
function getGeometryType(err, data){
if (err) throw err;
_.extend(req.params, {dbname:data});
cartoData.getDatabase(req, this);
},
function getGeometryType(err, data){
if (err) throw err;
_.extend(req.params, {dbname:data});
cartoData.getGeometryType(req, this);
},
function finishSetup(err, data){
if (!_.isNull(data))
_.extend(req.params, {geom_type: data});
cartoData.getGeometryType(req, this);
},
function finishSetup(err, data){
if (!_.isNull(data))
_.extend(req.params, {geom_type: data});
callback(err, req);
}
);
};
callback(err, req);
}
);
};
/**
* Little helper method to get the current list of infowindow variables and return to client
* @param req
* @param callback
*/
me.getInfowindow = function(req, callback){
var that = this;
/**
* Little helper method to get the current list of infowindow variables and return to client
* @param req
* @param callback
*/
me.getInfowindow = function(req, callback){
var that = this;
Step(
function(){
that.req2params(req, this);
},
function(err, data){
if (err) throw err;
cartoData.getInfowindow(data, callback);
}
);
};
Step(
function(){
that.req2params(req, this);
},
function(err, data){
if (err) throw err;
cartoData.getInfowindow(data, callback);
}
);
};
/**
* Little helper method to get map metadata and return to client
* @param req
* @param callback
*/
me.getMapMetadata = function(req, callback){
var that = this;
/**
* Little helper method to get map metadata and return to client
* @param req
* @param callback
*/
me.getMapMetadata = function(req, callback){
var that = this;
Step(
function(){
that.req2params(req, this);
},
function(err, data){
if (err) throw err;
cartoData.getMapMetadata(data, callback);
}
);
};
Step(
function(){
that.req2params(req, this);
},
function(err, data){
if (err) throw err;
cartoData.getMapMetadata(data, callback);
}
);
};
return me;
/**
* Helper to clear out tile cache on request
* @param req
* @param callback
*/
me.flushCache = function(req, Cache, callback){
var that = this;
Step(
function(){
that.req2params(req, this);
},
function(err, data){
if (err) throw err;
Cache.invalidate_db(req.params.dbname, req.params.table);
callback(null, true);
}
);
};
return me;
}();

3
package.json
View File

@ -22,7 +22,8 @@
"step": "0.0.x",
"generic-pool": "1.0.x",
"redis": "0.6.7",
"hiredis": "0.1.12"
"hiredis": "0.1.12",
"request": "2.9.202"
},
"devDependencies": {
"expresso": "0.8.x"