cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add explain cache and cache status route

Browse Source
This commit is contained in:
Simon Tokumine 2012-05-01 16:46:30 +01:00
parent 977f7d0e79
commit 7e074bbc98
1 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
app/controllers/app.js
View File

@ -18,19 +18,25 @@ var express = require('express')
buffer: true,
format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
}))
, Step = require('step')
, csv = require('csv')
, Meta = require(global.settings.app_root + '/app/models/metadata')
, oAuth = require(global.settings.app_root + '/app/models/oauth')
, PSQL = require(global.settings.app_root + '/app/models/psql')
, ApiKeyAuth = require(global.settings.app_root + '/app/models/apikey_auth')
, _ = require('underscore');
, Step = require('step')
, csv = require('csv')
, crypto = require('crypto')
, Meta = require(global.settings.app_root + '/app/models/metadata')
, oAuth = require(global.settings.app_root + '/app/models/oauth')
, PSQL = require(global.settings.app_root + '/app/models/psql')
, ApiKeyAuth = require(global.settings.app_root + '/app/models/apikey_auth')
, _ = require('underscore')
, tableCache = {};
app.use(express.bodyParser());
app.enable('jsonp callback');
app.all('/api/v1/sql', function(req, res) { handleQuery(req, res) } );
// basic routing
app.all('/api/v1/sql', function(req, res) { handleQuery(req, res) } );
app.all('/api/v1/sql.:f', function(req, res) { handleQuery(req, res) } );
app.get('/api/v1/cachestatus', function(req, res) { handleCacheStatus(req, res) } );
// request handlers
function handleQuery(req, res){
// sanitize input
@ -53,7 +59,10 @@ function handleQuery(req, res){
// setup step run
var start = new Date().getTime();
// initialise MD5 key of sql for cache lookups
var sql_md5 = generateMD5(sql);
try {
if (!_.isString(sql)) throw new Error("You must indicate a sql query");
var pg, explain_result;
@ -81,14 +90,23 @@ function handleQuery(req, res){
// store postgres connection
pg = new PSQL(user_id, database, limit, offset);
// get all the tables
pg.query("SELECT CDB_QueryTables($$" + sql + "$$)", this);
// get all the tables from Cache or SQL
if (!_.isNull(tableCache[sql_md5]) && !_.isUndefined(tableCache[sql_md5])){
tableCache[sql_md5].hits++;
return true;
} else{
pg.query("SELECT CDB_QueryTables($$" + sql + "$$)", this);
}
},
function queryResult(err, result){
if (err) throw err;
// store explain result
explain_result = result;
// store explain result in local Cache
if (_.isUndefined(tableCache[sql_md5])){
tableCache[sql_md5] = result;
tableCache[sql_md5].hits = 1; //initialise hit counter
}
// TODO: refactor formats to external object
if (format === 'geojson'){
@ -110,7 +128,7 @@ function handleQuery(req, res){
// set cache headers
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age=3600,must-revalidate, public');
res.header('X-Cache-Channel', generateCacheKey(database,explain_result));
res.header('X-Cache-Channel', generateCacheKey(database, tableCache[sql_md5]));
return result;
},
@ -147,6 +165,15 @@ function handleQuery(req, res){
}
}
function handleCacheStatus(req, res){
var tableCacheValues = _.values(tableCache);
var totalExplainHits = _.reduce(tableCacheValues, function(memo, res) { return memo + res.hits}, 0);
var totalExplainKeys = tableCacheValues.length;
res.send({explain: {hits: totalExplainHits, keys : totalExplainKeys }});
}
// helper functions
function toGeoJSON(data, res, callback){
try{
var out = {
@ -218,6 +245,12 @@ function generateCacheKey(database,tables){
return database + ":" + tables.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
}
function generateMD5(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
function handleException(err, res){
var msg = (global.settings.environment == 'development') ? {error:[err.message], stack: err.stack} : {error:[err.message]}
if (global.settings.environment !== 'test'){