diff --git a/config/environments/development.js b/config/environments/development.js index b341e17d..f1dcaba8 100644 --- a/config/environments/development.js +++ b/config/environments/development.js @@ -22,8 +22,10 @@ var config = { reapIntervalMillis: 1 } ,sqlapi: { - host: '127.0.0.1', - port: 8080 + protocol: 'http', + host: 'localhost.lan', + port: 8080, + version: 'v1' } ,varnish: { host: 'localhost', diff --git a/config/environments/production.js b/config/environments/production.js index a722bcff..5d754aab 100644 --- a/config/environments/production.js +++ b/config/environments/production.js @@ -1,5 +1,5 @@ var config = { - environment: 'production' + environment: 'production' ,port: 8181 ,host: '127.0.0.1' ,enable_cors: true @@ -15,8 +15,10 @@ var config = { port: 6379 } ,sqlapi: { - host: '127.0.0.1', - port: 8080 + protocol: 'https', + host: 'cartodb.com', + port: 8080, + version: 'v2' } ,varnish: { host: 'localhost', diff --git a/config/environments/test.js b/config/environments/test.js index 20f0eead..1d16cdbf 100644 --- a/config/environments/test.js +++ b/config/environments/test.js @@ -19,8 +19,10 @@ var config = { reapIntervalMillis: 1 } ,sqlapi: { - host: '127.0.0.1', - port: 8080 + protocol: 'http', + host: 'localhost.lan', + port: 8080, + version: 'v1' } ,varnish: { host: '', diff --git a/lib/cartodb/cache_validator.js b/lib/cartodb/cache_validator.js index 3ff5886b..ac6cedcc 100644 --- a/lib/cartodb/cache_validator.js +++ b/lib/cartodb/cache_validator.js @@ -1,17 +1,88 @@ -var _ = require('underscore'), - Varnish = require('node-varnish'); - -var varnish_queue = null; +var _ = require('underscore'), + Varnish = require('node-varnish'), + request = require('request'), + crypto = require('crypto'), + channelCache = {}, + varnish_queue = null; function init(host, port) { varnish_queue = new Varnish.VarnishQueue(host, port); } -function invalidate_db(dbname) { - varnish_queue.run_cmd('purge obj.http.X-Cache-Channel == ' + dbname); +function invalidate_db(dbname, table) { + try{ + varnish_queue.run_cmd('purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"'); + console.log('[SUCCESS FLUSHING CACHE]'); + } catch (e) { + console.log("[ERROR FLUSHING CACHE] Is enable_cache set to true? Failed for: " + 'purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"'); + } +} + +function generateCacheChannel(req, callback){ + var cacheChannel = ""; + + // use key to call sql api with sql request if present, else just return dbname and table name + // base key + var tableNames = req.params.table; + var dbName = req.params.dbname; + var username = req.headers.host.split('.')[0]; + + // replace tableNames with the results of the explain if present + if (_.isString(req.params.sql) && req.params.sql != ''){ + // initialise MD5 key of sql for cache lookups + var sql_md5 = generateMD5(req.params.sql); + var api = global.environment.sqlapi; + var qs = {}; + + // use cache if present + if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) { + callback(channelCache[sql_md5]); + } else{ + // strip out windshaft/mapnik inserted sql + var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/)[1]; + + // build up api string + var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql' + + // add query to querystring + qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; + + // add api_key if present in tile request (means table is private) + if (_.isString(req.params.map_key) && req.params.map_key != ''){ + qs.api_key = req.params.map_key; + } + + // call sql api + request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){ + if (!err && response.statusCode == 200) { + tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1]; + } else { + //oops, no SQL API. Just cache using fallback 'table' key + tableNames = 'table'; + } + cacheChannel = buildCacheChannel(dbName,tableNames); + channelCache[sql_md5] = cacheChannel; // store for caching + callback(cacheChannel); + }); + } + } else { + cacheChannel = buildCacheChannel(dbName,tableNames); + callback(cacheChannel); + } +} + +function buildCacheChannel(dbName, tableNames){ + return dbName + ':' + tableNames; +} + +function generateMD5(data){ + var hash = crypto.createHash('md5'); + hash.update(data); + return hash.digest('hex'); } module.exports = { init: init, - invalidate_db: invalidate_db + invalidate_db: invalidate_db, + generateCacheChannel: generateCacheChannel } diff --git a/lib/cartodb/carto_data.js b/lib/cartodb/carto_data.js index dd27d8a1..6e219e7b 100644 --- a/lib/cartodb/carto_data.js +++ b/lib/cartodb/carto_data.js @@ -18,13 +18,12 @@ module.exports = function() { var me = { user_metadata_db: 5, table_metadata_db: 0, - user_key: "rails:users:<%= username %>", - map_key: "rails:users:<%= username %>:map_key", + user_key: "rails:users:<%= username %>", + map_key: "rails:users:<%= username %>:map_key", table_key: "rails:<%= database_name %>:<%= table_name %>" }; - /** * Get the database name for this particular subdomain/username * @@ -56,7 +55,7 @@ module.exports = function() { }; /** - * Get the user map key for this particular subdomain/username + * Check the user map key for this particular subdomain/username * * @param req - standard express req object. importantly contains host information * @param callback diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 2d8f4ac6..ba331fc3 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -8,17 +8,19 @@ var CartodbWindshaft = function(serverOptions) { // set the cache chanel info to invalidate the cache on the frontend server serverOptions.afterTileRender = function(req, res, tile, headers, callback) { - res.header('X-Cache-Channel', req.params.dbname); - res.header('Last-Modified', new Date().toUTCString()); - res.header('Cache-Control', 'no-cache,max-age=86400,must-revalidate, public'); - callback(null, tile, headers); + Cache.generateCacheChannel(req, function(channel){ + res.header('X-Cache-Channel', channel); + res.header('Last-Modified', new Date().toUTCString()); + res.header('Cache-Control', 'no-cache,max-age=86400,must-revalidate, public'); + callback(null, tile, headers); + }); }; if(serverOptions.cache_enabled) { console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); Cache.init(serverOptions.varnish_host, serverOptions.varnish_port); serverOptions.afterStateChange = function(req, data, callback) { - Cache.invalidate_db(req.params.dbname); + Cache.invalidate_db(req.params.dbname, req.params.table); callback(null, data); } } @@ -65,6 +67,24 @@ var CartodbWindshaft = function(serverOptions) { ); }); + /** + * Helper API to allow per table tile cache (and sql cache) to be invalidated remotely. + * TODO: Move? + */ + ws.del(serverOptions.base_url + '/flush_cache', function(req, res){ + Step( + function(){ + serverOptions.flushCache(req, Cache, this); + }, + function(err, data){ + if (err){ + res.send(500); + } else { + res.send({status: 'ok'}, 200); + } + } + ); + }); return ws; } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 7ade5e8b..0e52c9ad 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -10,96 +10,117 @@ module.exports = function(){ enable_cors: global.environment.enable_cors, varnish_host: global.environment.varnish.host, varnish_port: global.environment.varnish.port, + cache_enabled: global.environment.cache_enabled, log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m' -}; + }; -/** - * Whitelist input and get database name & default geometry type from - * subdomain/user metadata held in CartoDB Redis - * @param req - standard express request obj. Should have host & table - * @param callback - */ -me.req2params = function(req, callback){ + /** + * Whitelist input and get database name & default geometry type from + * subdomain/user metadata held in CartoDB Redis + * @param req - standard express request obj. Should have host & table + * @param callback + */ + me.req2params = function(req, callback){ - // Whitelist query parameters and attach format - var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style']; - var bad_query = _.difference(_.keys(req.query), good_query); + // Whitelist query parameters and attach format + var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style']; + var bad_query = _.difference(_.keys(req.query), good_query); - _.each(bad_query, function(key){ delete req.query[key]; }); - req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object + _.each(bad_query, function(key){ delete req.query[key]; }); + req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object - // bring all query values onto req.params object - _.extend(req.params, req.query); + // bring all query values onto req.params object + _.extend(req.params, req.query); - // for cartodb, ensure interactivity is cartodb_id or user specified - req.params.interactivity = req.params.interactivity || 'cartodb_id'; + // for cartodb, ensure interactivity is cartodb_id or user specified + req.params.interactivity = req.params.interactivity || 'cartodb_id'; - Step( - function getPrivacy(){ - cartoData.authorize(req, this); - }, - function gatekeep(err, data){ - if(err) throw err; - if(data === "0") throw new Error("Sorry, you are unauthorized"); - return data; - }, - function getDatabase(err, data){ - if(err) throw err; + Step( + function getPrivacy(){ + cartoData.authorize(req, this); + }, + function gatekeep(err, data){ + if(err) throw err; + if(data === "0") throw new Error("Sorry, you are unauthorized"); + return data; + }, + function getDatabase(err, data){ + if(err) throw err; - cartoData.getDatabase(req, this); - }, - function getGeometryType(err, data){ - if (err) throw err; - _.extend(req.params, {dbname:data}); + cartoData.getDatabase(req, this); + }, + function getGeometryType(err, data){ + if (err) throw err; + _.extend(req.params, {dbname:data}); - cartoData.getGeometryType(req, this); - }, - function finishSetup(err, data){ - if (!_.isNull(data)) - _.extend(req.params, {geom_type: data}); + cartoData.getGeometryType(req, this); + }, + function finishSetup(err, data){ + if (!_.isNull(data)) + _.extend(req.params, {geom_type: data}); - callback(err, req); - } - ); -}; + callback(err, req); + } + ); + }; -/** - * Little helper method to get the current list of infowindow variables and return to client - * @param req - * @param callback - */ -me.getInfowindow = function(req, callback){ - var that = this; + /** + * Little helper method to get the current list of infowindow variables and return to client + * @param req + * @param callback + */ + me.getInfowindow = function(req, callback){ + var that = this; - Step( - function(){ - that.req2params(req, this); - }, - function(err, data){ - if (err) throw err; - cartoData.getInfowindow(data, callback); - } - ); -}; + Step( + function(){ + that.req2params(req, this); + }, + function(err, data){ + if (err) throw err; + cartoData.getInfowindow(data, callback); + } + ); + }; -/** - * Little helper method to get map metadata and return to client - * @param req - * @param callback - */ -me.getMapMetadata = function(req, callback){ - var that = this; + /** + * Little helper method to get map metadata and return to client + * @param req + * @param callback + */ + me.getMapMetadata = function(req, callback){ + var that = this; - Step( - function(){ - that.req2params(req, this); - }, - function(err, data){ - if (err) throw err; - cartoData.getMapMetadata(data, callback); - } - ); -}; + Step( + function(){ + that.req2params(req, this); + }, + function(err, data){ + if (err) throw err; + cartoData.getMapMetadata(data, callback); + } + ); + }; -return me; + /** + * Helper to clear out tile cache on request + * @param req + * @param callback + */ + me.flushCache = function(req, Cache, callback){ + var that = this; + + Step( + function(){ + that.req2params(req, this); + }, + function(err, data){ + if (err) throw err; + Cache.invalidate_db(req.params.dbname, req.params.table); + callback(null, true); + } + ); + }; + + return me; }(); diff --git a/package.json b/package.json index 64714bd7..f92e898e 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.6.7", - "hiredis": "0.1.12" + "hiredis": "0.1.12", + "request": "2.9.202" }, "devDependencies": { "expresso": "0.8.x"