cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cache channel now in layergroup controller

Browse Source 
Internal cache channel dbname+layergroupid cache must be unified in layergroup
and map controllers
Removes sendWithHeaders
This commit is contained in:
Raul Ochoa 2015-07-14 11:55:49 +02:00
parent 36257f73b9
commit c295584864
4 changed files with 157 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
lib/cartodb/controllers/layergroup.js
View File

@ -11,16 +11,21 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'
* @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend
* @param {{UserLimitsApi}} userLimitsApi
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
* @constructor
*/
function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi) {
function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi,
queryTablesApi) {
this.app = app;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.channelCache = {};
}
module.exports = LayergroupController;
@ -62,7 +67,7 @@ LayergroupController.prototype.attributes = function(req, res) {
var statusCode = self.app.findStatusCode(err);
self.app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err);
} else {
self.app.sendResponse(res, [tile, 200]);
self.sendResponse(req, res, [tile, 200]);
}
}
);
@ -149,7 +154,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
global.statsClient.increment('windshaft.tiles.error');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
} else {
this.app.sendWithHeaders(res, tile, 200, headers);
this.sendResponse(req, res, [tile, headers, 200]);
global.statsClient.increment('windshaft.tiles.success');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
}
@ -206,8 +211,120 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
self.app.sendError(res, {errors: ['' + err] }, self.app.findStatusCode(err), 'STATIC_MAP', err);
} else {
res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format);
self.app.sendResponse(res, [image, 200]);
self.sendResponse(req, res, [image, 200]);
}
}
);
};
LayergroupController.prototype.sendResponse = function(req, res, args) {
var self = this;
res.header('Cache-Control', 'public,max-age=31536000');
// Set Last-Modified header
var lastUpdated;
if (req.params.cache_buster) {
// Assuming cache_buster is a timestamp
lastUpdated = new Date(parseInt(req.params.cache_buster));
} else {
lastUpdated = new Date();
}
res.header('Last-Modified', lastUpdated.toUTCString());
step(
function getCacheChannel() {
self.cacheChannel(req, this);
},
function sendResponse(err, cacheChannel) {
if (err) {
console.log('ERROR generating cache channel: ' + err);
}
if (!!cacheChannel) {
res.header('X-Cache-Channel', cacheChannel);
}
self.app.sendResponse(res, args);
}
);
};
LayergroupController.prototype.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};
LayergroupController.prototype.cacheChannel = function(req, callback) {
if (req.profiler) {
req.profiler.start('addCacheChannel');
}
var dbName = req.params.dbname;
// no token means no tables associated
if (!req.params.token) {
return callback(null, this.buildCacheChannel(dbName, []));
}
var self = this;
var cacheKey = [ dbName, req.params.token ].join(':');
step(
function checkCached() {
if ( self.channelCache.hasOwnProperty(cacheKey) ) {
return callback(null, self.channelCache[cacheKey]);
}
return null;
},
function extractSQL(err) {
assert.ifError(err);
step(
function loadFromStore() {
self.mapStore.load(req.params.token, this);
},
function getSQL(err, mapConfig) {
if (req.profiler) {
req.profiler.done('mapStore_load');
}
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
assert.ifError(err);
if (req.profiler) {
req.profiler.done('affectedTables');
}
var cacheChannel = self.buildCacheChannel(dbName, tableNames);
self.channelCache[cacheKey] = cacheChannel;
return cacheChannel;
},
function finish(err, cacheChannel) {
callback(err, cacheChannel);
}
);
};

10
lib/cartodb/controllers/map.js
View File

@ -37,6 +37,8 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
this.channelCache = {};
}
module.exports = MapController;
@ -274,8 +276,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
var cacheChannel = self.app.buildCacheChannel(dbName, result.affectedTables);
self.app.channelCache[cacheKey] = cacheChannel;
var cacheChannel = self.buildCacheChannel(dbName, result.affectedTables);
self.channelCache[cacheKey] = cacheChannel;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
@ -298,3 +300,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
}
);
};
MapController.prototype.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};

2
lib/cartodb/controllers/named_maps.js
View File

@ -63,7 +63,7 @@ NamedMapsController.prototype.tile = function(req, res) {
self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName()));
res.setHeader('Content-Type', headers['Content-Type']);
res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate');
self.app.sendWithHeaders(res, tile, 200, headers);
self.app.sendResponse(res, [tile, headers, 200]);
}
}
);

252
lib/cartodb/server.js
View File

@ -74,12 +74,6 @@ module.exports = function(serverOptions) {
max_user_templates: global.environment.maxUserTemplates
});
// This is for Templated maps
//
// "named" is the official, "template" is for backward compatibility up to 1.6.x
//
var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
var surrogateKeysCacheBackends = [];
if (serverOptions.varnish_purge_enabled) {
@ -197,7 +191,8 @@ module.exports = function(serverOptions) {
tileBackend,
previewBackend,
attributesBackend,
userLimitsApi
userLimitsApi,
queryTablesApi
).register(app);
new controller.Map(
@ -243,98 +238,33 @@ module.exports = function(serverOptions) {
}
});
// GET routes for which we don't want to request any caching.
// POST/PUT/DELETE requests are never cached anyway.
var noCacheGETRoutes = [
'/',
'/version',
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
serverOptions.base_url_mapconfig,
serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
template_baseurl,
template_baseurl + '/:template_id',
template_baseurl + '/:template_id/jsonp'
];
app.sendResponse = function(res, args) {
var statusCode;
if ( res._windshaftStatusCode ) {
// Added by our override of sendError
statusCode = res._windshaftStatusCode;
} else {
if ( args.length > 2 ) statusCode = args[2];
else {
statusCode = args[1] || 200;
var req = res.req;
if (global.environment && global.environment.api_hostname) {
res.header('X-Served-By-Host', global.environment.api_hostname);
}
if (req && req.params && req.params.dbhost) {
res.header('X-Served-By-DB-Host', req.params.dbhost);
}
if ( req && req.profiler ) {
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
// res.send(body|status[, headers|status[, status]])
res.send.apply(res, args);
if ( req && req.profiler ) {
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
console.error("error sending profiling stats: " + err);
}
}
var req = res.req;
step (
function addCacheChannel() {
if ( ! req ) {
// having no associated request can happen when
// using fake response objects for testing layergroup
// creation
return false;
}
if ( ! req.params ) {
// service requests (/version, /)
// have no need for an X-Cache-Channel
return false;
}
if ( statusCode != 200 ) {
// We do not want to cache
// unsuccessful responses
return false;
}
if ( _.contains(noCacheGETRoutes, req.route.path) ) {
//console.log("Skipping cache channel in route:\n" + req.route.path);
return false;
}
//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" +
// mapCreateRoutes.join("\n"));
app.addCacheChannel(req, this);
},
function sendResponse(err/*, added*/) {
if ( err ) console.log(err + err.stack);
// When using custom results from tryFetch* methods,
// there is no "req" link in the result object.
// In those cases we don't want to send stats now
// as they will be sent at the real end of request
var req = res.req;
if (global.environment && global.environment.api_hostname) {
res.header('X-Served-By-Host', global.environment.api_hostname);
}
if (req && req.params && req.params.dbhost) {
res.header('X-Served-By-DB-Host', req.params.dbhost);
}
if ( req && req.profiler ) {
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
res.send.apply(res, args);
if ( req && req.profiler ) {
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
console.error("error sending profiling stats: " + err);
}
}
return null;
},
function finish(err) {
if ( err ) console.log(err + err.stack);
}
);
};
app.sendWithHeaders = function(res, what, status, headers) {
app.sendResponse(res, [what, headers, status]);
};
app.sendError = function(res, err, statusCode, label, tolog) {
@ -490,136 +420,6 @@ module.exports = function(serverOptions) {
);
};
// TODO: review lifetime of elements of this cache
// NOTE: by-token indices should only be dropped when
// the corresponding layegroup is dropped, because
// we have no SQL after layer creation.
app.channelCache = {};
app.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};
app.generateCacheChannel = function(app, req, callback){
// Build channelCache key
var dbName = req.params.dbname;
var cacheKey = [ dbName, req.params.token ].join(':');
// no token means no tables associated
if (!req.params.token) {
return callback(null, this.buildCacheChannel(dbName, []));
}
step(
function checkCached() {
if ( app.channelCache.hasOwnProperty(cacheKey) ) {
return callback(null, app.channelCache[cacheKey]);
}
return null;
},
function extractSQL(err) {
assert.ifError(err);
step(
function loadFromStore() {
mapStore.load(req.params.token, this);
},
function getSQL(err, mapConfig) {
if (req.profiler) {
req.profiler.done('mapStore_load');
}
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
assert.ifError(err);
if (req.profiler) {
req.profiler.done('affectedTables');
}
var cacheChannel = app.buildCacheChannel(dbName,tableNames);
app.channelCache[cacheKey] = cacheChannel;
return cacheChannel;
},
function finish(err, cacheChannel) {
callback(err, cacheChannel);
}
);
};
// Set the cache chanel info to invalidate the cache on the frontend server
//
// @param req The request object.
// The function will have no effect unless req.res exists.
// It is expected that req.params contains 'table' and 'dbname'
//
// @param cb function(err, channel) will be called when ready.
// the channel parameter will be null if nothing was added
//
app.addCacheChannel = function(req, cb) {
// skip non-GET requests, or requests for which there's no response
if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
if (req.profiler) {
req.profiler.start('addCacheChannel');
}
var res = req.res;
if ( req.params.token ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
var ttl = global.environment.varnish.ttl || 86400;
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
}
// Set Last-Modified header
var lastUpdated;
if ( req.params.cache_buster ) {
// Assuming cache_buster is a timestamp
// FIXME: store lastModified in the cache channel instead
lastUpdated = new Date(parseInt(req.params.cache_buster));
} else {
lastUpdated = new Date();
}
res.header('Last-Modified', lastUpdated.toUTCString());
app.generateCacheChannel(app, req, function(err, channel){
if (req.profiler) {
req.profiler.done('generateCacheChannel');
req.profiler.end();
}
if ( ! err ) {
res.header('X-Cache-Channel', channel);
cb(null, channel);
} else {
console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
// TODO: evaluate if we should bubble up the error instead
cb(null, 'ERROR');
}
});
};
return app;
};