diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index ebafbbac..e0850822 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -6,12 +6,20 @@ var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.mapConfigAdapter = mapConfigAdapter; + this.affectedTablesCache = affectedTablesCache; this.providerCache = new LruCache({ max: 2000 }); } @@ -30,6 +38,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.metadataBackend, this.userLimitsApi, this.mapConfigAdapter, + this.affectedTablesCache, user, templateId, config, diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 791f147d..f632b394 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -9,6 +9,8 @@ const authorize = require('../middleware/authorize'); const dbConnSetup = require('../middleware/db-conn-setup'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheControlHeader = require('../middleware/cache-control-header'); +const sendResponse = require('../middleware/send-response'); function AnalysesController(pgConnection, authApi, userLimitsApi) { this.pgConnection = pgConnection; @@ -36,7 +38,7 @@ AnalysesController.prototype.register = function (app) { getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), prepareResponse(), - setCacheControlHeader(), + cacheControlHeader({ ttl: 10, revalidate: true }), sendResponse(), unauthorizedError() ); @@ -111,25 +113,6 @@ function prepareResponse () { }; } -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - }; -} - function unauthorizedError () { return function unathorizedErrorMiddleware(err, req, res, next) { if (err.message.match(/permission\sdenied/)) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index eb45c022..a77a7168 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -9,10 +9,14 @@ const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); +const sendResponse = require('../middleware/send-response'); const DataviewBackend = require('../backends/dataview'); const AnalysisStatusBackend = require('../backends/analysis-status'); const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); -const QueryTables = require('cartodb-query-tables'); const SUPPORTED_FORMATS = { grid_json: true, json_torque: true, @@ -43,7 +47,7 @@ function LayergroupController( attributesBackend, surrogateKeysCache, userLimitsApi, - layergroupAffectedTables, + layergroupAffectedTablesCache, analysisBackend, authApi ) { @@ -54,7 +58,7 @@ function LayergroupController( this.attributesBackend = attributesBackend; this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.dataviewBackend = new DataviewBackend(analysisBackend); this.analysisStatusBackend = new AnalysisStatusBackend(); @@ -64,10 +68,10 @@ function LayergroupController( module.exports = LayergroupController; LayergroupController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath } = app; + const { base_url_mapconfig: mapConfigBasePath } = app; app.get( - `${mapconfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, + `${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, cors(), cleanUpQueryParams(), locals(), @@ -77,13 +81,17 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -92,7 +100,7 @@ LayergroupController.prototype.register = function(app) { ); app.get( - `${mapconfigBasePath}/:token/:z/:x/:y.:format`, + `${mapConfigBasePath}/:token/:z/:x/:y.:format`, cors(), cleanUpQueryParams(), locals(), @@ -102,13 +110,17 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -117,7 +129,7 @@ LayergroupController.prototype.register = function(app) { ); app.get( - `${mapconfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, + `${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, distinguishLayergroupFromStaticRoute(), cors(), cleanUpQueryParams(), @@ -128,13 +140,17 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getTile(this.tileBackend, 'maplayer_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -143,7 +159,7 @@ LayergroupController.prototype.register = function(app) { ); app.get( - `${mapconfigBasePath}/:token/:layer/attributes/:fid`, + `${mapConfigBasePath}/:token/:layer/attributes/:fid`, cors(), cleanUpQueryParams(), locals(), @@ -153,20 +169,24 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getFeatureAttributes(this.attributesBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); const forcedFormat = 'png'; app.get( - `${mapconfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, + `${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, cors(), cleanUpQueryParams(['layer']), locals(), @@ -176,18 +196,23 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), getPreviewImageByCenter(this.previewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); app.get( - `${mapconfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, + `${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, cors(), cleanUpQueryParams(['layer']), locals(), @@ -197,13 +222,18 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), getPreviewImageByBoundingBox(this.previewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -226,7 +256,7 @@ LayergroupController.prototype.register = function(app) { ]; app.get( - `${mapconfigBasePath}/:token/dataview/:dataviewName`, + `${mapConfigBasePath}/:token/dataview/:dataviewName`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -236,18 +266,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getDataview(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/:layer/widget/:dataviewName`, + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -257,18 +291,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getDataview(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/dataview/:dataviewName/search`, + `${mapConfigBasePath}/:token/dataview/:dataviewName/search`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -278,18 +316,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/:layer/widget/:dataviewName/search`, + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -299,18 +341,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/analysis/node/:nodeId`, + `${mapConfigBasePath}/:token/analysis/node/:nodeId`, cors(), cleanUpQueryParams(), locals(), @@ -366,7 +412,13 @@ function getRequestParams(locals) { return params; } -function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) { +function createMapStoreMapConfigProvider ( + mapStore, + userLimitsApi, + pgConnection, + affectedTablesCache, + forcedFormat = null +) { return function createMapStoreMapConfigProviderMiddleware (req, res, next) { const { user } = res.locals; @@ -377,7 +429,14 @@ function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat params.layer = params.layer || 'all'; } - res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params); + res.locals.mapConfigProvider = new MapStoreMapConfigProvider( + mapStore, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + params + ); next(); }; @@ -552,110 +611,6 @@ function getPreviewImageByBoundingBox (previewBackend) { }; } -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware (req, res, next) { - let { cache_buster: cacheBuster } = res.locals; - - cacheBuster = parseInt(cacheBuster, 10); - - const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date(); - - res.set('Last-Modified', lastUpdated.toUTCString()); - - next(); - }; -} - -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=31536000'); - - next(); - }; -} - -function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) { - return function getAffectedTablesMiddleware (req, res, next) { - const { user, dbname, token } = res.locals; - - if (layergroupAffectedTables.hasAffectedTables(dbname, token)) { - res.locals.affectedTables = layergroupAffectedTables.get(dbname, token); - return next(); - } - - mapStore.load(token, (err, mapconfig) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - const queries = []; - mapconfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - queries.push(`SELECT * FROM ${table} LIMIT 0`); - }); - } - }); - - const sql = queries.length ? queries.join(';') : null; - - if (!sql) { - global.logger.warn('ERROR generating cache channel:' + - ' this request doesn\'t need an X-Cache-Channel generated'); - return next(); - } - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - global.logger.warn('ERROR generating cache channel: ', err); - return next(); - } - - // feed affected tables cache so it can be reused from, for instance, map controller - layergroupAffectedTables.set(dbname, token, affectedTables); - - res.locals.affectedTables = affectedTables; - - next(); - }); - }); - }); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - surrogateKeysCache.tag(res, affectedTables); - } - - next(); - }; -} - function incrementSuccessMetrics (statsClient) { return function incrementSuccessMetricsMiddleware (req, res, next) { const formatStat = parseFormat(req.params.format); @@ -667,24 +622,6 @@ function incrementSuccessMetrics (statsClient) { }; } -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - - res.status(res.statusCode || 200); - - if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - } else { - res.send(res.body); - } - }; -} - function incrementErrorMetrics (statsClient) { return function incrementErrorMetricsMiddleware (err, req, res, next) { const formatStat = parseFormat(req.params.format); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 71229c51..cdb1515f 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,7 +2,6 @@ const _ = require('underscore'); const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const QueryTables = require('cartodb-query-tables'); const ResourceLocator = require('../models/resource-locator'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); @@ -12,7 +11,11 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); +const sendResponse = require('../middleware/send-response'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../utils/layergroup-metadata'); @@ -63,15 +66,15 @@ function MapController ( module.exports = MapController; MapController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app; + const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app; app.get( - `${mapconfigBasePath}`, + `${mapConfigBasePath}`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) ); app.post( - `${mapconfigBasePath}`, + `${mapConfigBasePath}`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) ); @@ -87,7 +90,7 @@ MapController.prototype.register = function(app) { this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) ); - app.options(app.base_url_mapconfig, cors('Content-Type')); + app.options(`${mapConfigBasePath}`, cors('Content-Type')); }; MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { @@ -112,11 +115,11 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us this.getCreateMapMiddlewares(useTemplate), incrementMapViewCount(this.metadataBackend), augmentLayergroupData(), - getAffectedTables(this.pgConnection, this.layergroupAffectedTables), - setCacheChannel(), - setLastModified(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), setLastUpdatedTimeToLayergroup(), - setCacheControl(), setLayerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), @@ -124,7 +127,6 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us setTurboCartoMetadataToLayergroup(this.layergroupMetadata), setAggregationMetadataToLayergroup(this.layergroupMetadata), setTilejsonMetadataToLayergroup(this.layergroupMetadata), - setSurrogateKeyHeader(this.surrogateKeysCache), sendResponse(), augmentError({ label, addContext }) ]; @@ -139,16 +141,27 @@ MapController.prototype.getCreateMapMiddlewares = function (useTemplate) { this.pgConnection, this.metadataBackend, this.userLimitsApi, - this.mapConfigAdapter + this.mapConfigAdapter, + this.layergroupAffectedTables ), - instantiateLayergroup(this.mapBackend, this.userLimitsApi) + instantiateLayergroup( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; } return [ checkCreateLayergroup(), prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup (this.mapBackend, this.userLimitsApi) + createLayergroup ( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; }; @@ -219,17 +232,25 @@ function checkCreateLayergroup () { }; } -function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function getTemplate ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { return function getTemplateMiddleware (req, res, next) { const templateParams = req.body; const { user } = res.locals; - const mapconfigProvider = new NamedMapMapConfigProvider( + const mapConfigProvider = new NamedMapMapConfigProvider( templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, + affectedTablesCache, user, req.params.template_id, templateParams, @@ -237,15 +258,15 @@ function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi res.locals ); - mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { req.profiler.done('named.getMapConfig'); if (err) { return next(err); } - res.locals.mapconfig = mapconfig; + res.locals.mapConfig = mapConfig; res.locals.rendererParams = rendererParams; - res.locals.mapconfigProvider = mapconfigProvider; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); @@ -288,51 +309,63 @@ function prepareAdapterMapConfig (mapConfigAdapter) { }; } -function createLayergroup (mapBackend, userLimitsApi) { +function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function createLayergroupMiddleware (req, res, next) { const requestMapConfig = req.body; const { context, user } = res.locals; const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapconfig = new MapConfig(requestMapConfig, datasource); - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals); + const mapConfig = new MapConfig(requestMapConfig, datasource); + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + res.locals + ); - res.locals.mapconfig = mapconfig; + res.locals.mapConfig = mapConfig; res.locals.analysesResults = context.analysesResults; - mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + mapBackend.createLayergroup(mapConfig, res.locals, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); }; } -function instantiateLayergroup (mapBackend, userLimitsApi) { +function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapconfig, rendererParams } = res.locals; - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams); + const { user, mapConfig, rendererParams } = res.locals; + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + rendererParams + ); - mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; - const { mapconfigProvider } = res.locals; + const { mapConfigProvider } = res.locals; - res.locals.analysesResults = mapconfigProvider.analysesResults; - res.locals.template = mapconfigProvider.template; - res.locals.templateName = mapconfigProvider.getTemplateName(); - res.locals.context = mapconfigProvider.context; + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; next(); }); @@ -341,10 +374,10 @@ function instantiateLayergroup (mapBackend, userLimitsApi) { function incrementMapViewCount (metadataBackend) { return function incrementMapViewCountMiddleware(req, res, next) { - const { mapconfig, user } = res.locals; + const { mapConfig, user } = res.locals; // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { req.profiler.done('incMapviewCount'); if (err) { @@ -358,7 +391,7 @@ function incrementMapViewCount (metadataBackend) { function augmentLayergroupData () { return function augmentLayergroupDataMiddleware (req, res, next) { - const { layergroup } = res.locals; + const layergroup = res.body; // include in layergroup response the variables in serverMedata // those variables are useful to send to the client information @@ -369,80 +402,33 @@ function augmentLayergroupData () { }; } -function getAffectedTables (pgConnection, layergroupAffectedTables) { - return function getAffectedTablesMiddleware (req, res, next) { - const { dbname, layergroup, user, mapconfig } = res.locals; +function setLastUpdatedTimeToLayergroup () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { mapConfigProvider, analysesResults } = res.locals; + const layergroup = res.body; - pgConnection.getConnection(user, (err, connection) => { + mapConfigProvider.getAffectedTables((err, affectedTables) => { if (err) { return next(err); } - const sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); + if (!affectedTables) { + return next(); + } - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - return next(err); - } + var lastUpdateTime = affectedTables.getLastUpdatedAt(); - // feed affected tables cache so it can be reused from, for instance, layergroup controller - layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - res.locals.affectedTables = affectedTables; + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - next(); - }); + next(); }); }; } -function setCacheChannel () { - return function setCacheChannelMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (req.method === 'GET') { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setLastModified () { - return function setLastModifiedMiddleware (req, res, next) { - if (req.method === 'GET') { - res.set('Last-Modified', (new Date()).toUTCString()); - } - - next(); - }; -} - -function setLastUpdatedTimeToLayergroup () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { affectedTables, layergroup, analysesResults } = res.locals; - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }; -} - function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { return lastUpdateTime; @@ -456,27 +442,17 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) { }, lastUpdateTime); } -function setCacheControl () { - return function setCacheControlMiddleware (req, res, next) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - } - - next(); - }; -} - function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { - const { user, mapconfig, layergroup } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; pgConnection.getConnection(user, (err, connection) => { if (err) { return next(err); } - statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { if (err) { return next(err); } @@ -495,7 +471,8 @@ function setLayerStats (pgConnection, statsBackend) { function setLayergroupIdHeader (templateMaps, useTemplateHash) { return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { layergroup, user, template } = res.locals; + const { user, template } = res.locals; + const layergroup = res.body; if (useTemplateHash) { var templateHash = templateMaps.fingerPrint(template).substring(0, 8); @@ -510,9 +487,10 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) { function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); next(); }; @@ -520,7 +498,8 @@ function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, user, analysesResults = [] } = res.locals; + const { user, analysesResults = [] } = res.locals; + const layergroup = res.body; layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); @@ -530,9 +509,10 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { function setTurboCartoMetadataToLayergroup (layergroupMetadata) { return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; + const layergroup = res.body; - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -540,9 +520,10 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) { function setAggregationMetadataToLayergroup (layergroupMetadata) { return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; + const layergroup = res.body; - layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -550,54 +531,24 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) { function setTilejsonMetadataToLayergroup (layergroupMetadata) { return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig); + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); next(); }; } -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { affectedTables, user, templateName } = res.locals; - - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTables); - } - - if (templateName) { - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } - }; -} - function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; return function augmentErrorMiddleware (err, req, res, next) { req.profiler.done('error'); - const { mapconfig } = res.locals; + const { mapConfig } = res.locals; if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; } err.label = label; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 815c68c8..26c5e4ad 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -1,4 +1,3 @@ -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); const locals = require('../middleware/locals'); @@ -7,6 +6,11 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); +const sendResponse = require('../middleware/send-response'); const vectorError = require('../middleware/vector-error'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -27,8 +31,8 @@ function getRequestParams(locals) { const params = Object.assign({}, locals); delete params.template; - delete params.affectedTablesAndLastUpdate; - delete params.namedMapProvider; + delete params.affectedTables; + delete params.mapConfigProvider; delete params.allowedQueryParams; return params; @@ -76,16 +80,15 @@ NamedMapsController.prototype.register = function(app) { namedMapProviderCache: this.namedMapProviderCache, label: 'NAMED_MAP_TILE' }), - getAffectedTables(), getTile({ tileBackend: this.tileBackend, label: 'NAMED_MAP_TILE' }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse(), vectorError() ); @@ -105,7 +108,6 @@ NamedMapsController.prototype.register = function(app) { namedMapProviderCache: this.namedMapProviderCache, label: 'STATIC_VIZ_MAP', forcedFormat: 'png' }), - getAffectedTables(), getTemplate({ label: 'STATIC_VIZ_MAP' }), prepareLayerFilterFromPreviewLayers({ namedMapProviderCache: this.namedMapProviderCache, @@ -113,12 +115,12 @@ NamedMapsController.prototype.register = function(app) { }), getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), - incrementMapViews({ metadataBackend: this.metadataBackend }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), + incrementMapViews({ metadataBackend: this.metadataBackend }), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); }; @@ -142,25 +144,7 @@ function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = nul return next(err); } - res.locals.namedMapProvider = namedMapProvider; - - next(); - }); - }; -} - -function getAffectedTables () { - return function getAffectedTables (req, res, next) { - const { namedMapProvider } = res.locals; - - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { - req.profiler.done('affectedTables'); - - if (err) { - return next(err); - } - - res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate; + res.locals.mapConfigProvider = namedMapProvider; next(); }); @@ -169,9 +153,9 @@ function getAffectedTables () { function getTemplate ({ label }) { return function getTemplateMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { mapConfigProvider } = res.locals; - namedMapProvider.getTemplate((err, template) => { + mapConfigProvider.getTemplate((err, template) => { if (err) { err.label = label; return next(err); @@ -219,7 +203,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) return next(err); } - res.locals.namedMapProvider = provider; + res.locals.mapConfigProvider = provider; next(); }); @@ -228,10 +212,11 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) function getTile ({ tileBackend, label }) { return function getTileMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { mapConfigProvider, format } = res.locals; - tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => { + tileBackend.getTile(mapConfigProvider, req.params, (err, tile, headers, stats) => { req.profiler.add(stats); + req.profiler.done('render-' + format); if (err) { err.label = label; @@ -251,7 +236,7 @@ function getTile ({ tileBackend, label }) { function getStaticImageOptions ({ tablesExtentApi }) { return function getStaticImageOptionsMiddleware(req, res, next) { - const { user, namedMapProvider, template } = res.locals; + const { user, mapConfigProvider, template } = res.locals; const imageOpts = getImageOptions(res.locals, template); @@ -262,18 +247,18 @@ function getStaticImageOptions ({ tablesExtentApi }) { res.locals.imageOpts = DEFAULT_ZOOM_CENTER; - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { + mapConfigProvider.getAffectedTables((err, affectedTables) => { if (err) { return next(); } - var affectedTables = affectedTablesAndLastUpdate.tables || []; + var tables = affectedTables.tables || []; - if (affectedTables.length === 0) { + if (tables.length === 0) { return next(); } - tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => { + tablesExtentApi.getBounds(user, tables, (err, bounds) => { if (err) { return next(); } @@ -353,7 +338,7 @@ function getImageOptionsFromBoundingBox (bbox = '') { function getImage({ previewBackend, label }) { return function getImageMiddleware (req, res, next) { - const { imageOpts, namedMapProvider } = res.locals; + const { imageOpts, mapConfigProvider } = res.locals; const { zoom, center, bounds } = imageOpts; let { width, height } = req.params; @@ -364,7 +349,7 @@ function getImage({ previewBackend, label }) { const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; if (zoom !== undefined && center) { - return previewBackend.getImage(namedMapProvider, format, width, height, zoom, center, + return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center, (err, image, headers, stats) => { req.profiler.add(stats); @@ -383,8 +368,9 @@ function getImage({ previewBackend, label }) { }); } - previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => { + previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => { req.profiler.add(stats); + req.profiler.done('render-' + format); if (err) { err.label = label; @@ -402,15 +388,23 @@ function getImage({ previewBackend, label }) { }; } +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} + function incrementMapViewsError (ctx) { return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`; } function incrementMapViews ({ metadataBackend }) { return function incrementMapViewsMiddleware(req, res, next) { - const { user, namedMapProvider } = res.locals; + const { user, mapConfigProvider } = res.locals; - namedMapProvider.getMapConfig((err, mapConfig) => { + mapConfigProvider.getMapConfig((err, mapConfig) => { if (err) { global.logger.log(incrementMapViewsError({ user, err })); return next(); @@ -458,84 +452,3 @@ function templateBounds(view) { } return false; } - -function setSurrogateKeyHeader ({ surrogateKeysCache }) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals; - - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName())); - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - if (affectedTablesAndLastUpdate.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTablesAndLastUpdate); - } - } - - next(); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel()); - } - - next(); - }; -} - -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - var lastModifiedDate; - if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) { - lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt()); - } else { - lastModifiedDate = new Date(); - } - - res.set('Last-Modified', lastModifiedDate.toUTCString()); - } - - next(); - }; - } - -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - // we increase cache control as we can invalidate it - res.set('Cache-Control', 'public,max-age=31536000'); - } - - next(); - }; - } - -function setContentTypeHeader () { - return function setContentTypeHeaderMiddleware(req, res, next) { - - res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - const { format } = res.locals; - - req.profiler.done('render-' + format); - - res.status(200); - res.send(res.body); - }; -} diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index db503fb7..136e9df7 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -5,6 +5,7 @@ const locals = require('../middleware/locals'); const credentials = require('../middleware/credentials'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const sendResponse = require('../middleware/send-response'); /** * @param {AuthApi} authApi @@ -224,12 +225,3 @@ function listTemplates ({ templateMaps }) { }); }; } - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(res.statusCode || 200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method](res.body); - }; -} diff --git a/lib/cartodb/middleware/cache-channel-header.js b/lib/cartodb/middleware/cache-channel-header.js new file mode 100644 index 00000000..d6bf394b --- /dev/null +++ b/lib/cartodb/middleware/cache-channel-header.js @@ -0,0 +1,24 @@ +module.exports = function setCacheChannelHeader () { + return function setCacheChannelHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider } = res.locals; + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Cache Channel Header:', err); + return next(); + } + + if (!affectedTables) { + return next(); + } + + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/middleware/cache-control-header.js new file mode 100644 index 00000000..574ae3b5 --- /dev/null +++ b/lib/cartodb/middleware/cache-control-header.js @@ -0,0 +1,19 @@ +const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365; + +module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) { + return function setCacheControlHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const directives = [ 'public', `max-age=${ttl}` ]; + + if (revalidate) { + directives.push('must-revalidate'); + } + + res.set('Cache-Control', directives.join(',')); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/last-modified-header.js b/lib/cartodb/middleware/last-modified-header.js new file mode 100644 index 00000000..18c0d961 --- /dev/null +++ b/lib/cartodb/middleware/last-modified-header.js @@ -0,0 +1,45 @@ +module.exports = function setLastModifiedHeader ({ now = false } = {}) { + return function setLastModifiedHeaderMiddleware(req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider, cache_buster } = res.locals; + + if (cache_buster) { + const cacheBuster = parseInt(cache_buster, 10); + const lastModifiedDate = Number.isFinite(cacheBuster) ? new Date(cacheBuster) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + return next(); + } + + // REVIEW: to keep 100% compatibility with maps controller + if (now) { + res.set('Last-Modified', new Date().toUTCString()); + + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Last Modified Header:', err); + return next(); + } + + if (!affectedTables) { + res.set('Last-Modified', new Date().toUTCString()); + + return next(); + } + + const lastUpdatedAt = affectedTables.getLastUpdatedAt(); + const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? new Date(lastUpdatedAt) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/send-response.js b/lib/cartodb/middleware/send-response.js new file mode 100644 index 00000000..469cf0a7 --- /dev/null +++ b/lib/cartodb/middleware/send-response.js @@ -0,0 +1,17 @@ +module.exports = function sendResponse () { + return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); + + res.status(res.statusCode || 200); + + if (Buffer.isBuffer(res.body)) { + return res.send(res.body); + } + + if (req.query.callback) { + return res.jsonp(res.body); + } + + res.json(res.body); + }; +}; diff --git a/lib/cartodb/middleware/surrogate-key-header.js b/lib/cartodb/middleware/surrogate-key-header.js new file mode 100644 index 00000000..51cec1c1 --- /dev/null +++ b/lib/cartodb/middleware/surrogate-key-header.js @@ -0,0 +1,31 @@ +const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); + +module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { user, mapConfigProvider } = res.locals; + + if (mapConfigProvider instanceof NamedMapMapConfigProvider) { + surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName())); + } + + if (req.method !== 'GET') { + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Surrogate Key Header:', err); + return next(); + } + + if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) { + return next(); + } + + surrogateKeysCache.tag(res, affectedTables); + + next(); + }); + }; +}; diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 340073b5..b3b3f191 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -2,6 +2,7 @@ var assert = require('assert'); var step = require('step'); var MapStoreMapConfigProvider = require('./map-store-provider'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapConfig} mapConfig @@ -11,10 +12,13 @@ var MapStoreMapConfigProvider = require('./map-store-provider'); * @constructor * @type {CreateLayergroupMapConfigProvider} */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) { + +function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapConfig = mapConfig; this.user = user; this.userLimitsApi = userLimitsApi; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.params = params; this.cacheBuster = params.cache_buster || 0; } @@ -23,7 +27,13 @@ module.exports = CreateLayergroupMapConfigProvider; CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig && this.params && this.context) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -31,6 +41,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { function handleRenderLimits(err, renderLimits) { assert.ifError(err); context.limits = renderLimits; + self.context = context; return null; }, function finish(err) { @@ -46,3 +57,52 @@ CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigPr CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter; CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey; + +CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + const queries = []; + + this.mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(null, affectedTables); + }); + }); + }); +}; diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index 177322d4..f89a79ce 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var dot = require('dot'); var step = require('step'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapStore} mapStore @@ -11,20 +12,30 @@ var step = require('step'); * @constructor * @type {MapStoreMapConfigProvider} */ -function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) { +function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapStore = mapStore; this.user = user; this.userLimitsApi = userLimitsApi; - this.params = params; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.token = params.token; this.cacheBuster = params.cache_buster || 0; + this.mapConfig = null; + this.params = params; + this.context = null; } module.exports = MapStoreMapConfigProvider; MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig !== null) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -39,6 +50,8 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { self.mapStore.load(self.token, this); }, function finish(err, mapConfig) { + self.mapConfig = mapConfig; + self.context = context; return callback(err, mapConfig, self.params, context); } ); @@ -74,4 +87,54 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) { scale_factor: 1 }); return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); -}; \ No newline at end of file +}; + +MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + + return callback(null, affectedTables); + } + + const queries = []; + + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); +}; diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 9b085dbc..50612065 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -11,8 +11,19 @@ var QueryTables = require('cartodb-query-tables'); * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, - owner, templateId, config, authToken, params) { +function NamedMapMapConfigProvider( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache, + owner, + templateId, + config, + authToken, + params +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; @@ -30,7 +41,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, // use template after call to mapConfig this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTablesCache = affectedTablesCache; // providing this.err = null; @@ -189,7 +200,7 @@ NamedMapMapConfigProvider.prototype.getCacheBuster = function() { NamedMapMapConfigProvider.prototype.reset = function() { this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTables = null; this.err = null; this.mapConfig = null; @@ -251,39 +262,51 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() { return this.templateName; }; -NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) { - var self = this; - - if (this.affectedTablesAndLastUpdate !== null) { - return callback(null, this.affectedTablesAndLastUpdate); - } - - step( - function getMapConfig() { - self.getMapConfig(this); - }, - function getSql(err, mapConfig) { - assert.ifError(err); - return mapConfig.getLayers().map(function(layer) { - return layer.options.sql; - }).join(';'); - }, - function getAffectedTables(err, sql) { - assert.ifError(err); - step( - function getConnection() { - self.pgConnection.getConnection(self.owner, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function finish(err, result) { - self.affectedTablesAndLastUpdate = result; - return callback(err, result); +NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); } - ); + + const { dbname } = this.rendererParams; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + const queries = []; + + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.owner, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7f7e2712..8535783d 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -200,7 +200,8 @@ module.exports = function(serverOptions) { pgConnection, metadataBackend, userLimitsApi, - mapConfigAdapter + mapConfigAdapter, + layergroupAffectedTablesCache ); ['update', 'delete'].forEach(function(eventType) { diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index dfb6174f..b4840879 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1272,6 +1272,8 @@ describe(suiteName, function() { it("cache control for layergroup default value", function(done) { global.environment.varnish.layergroupTtl = null; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=86400,must-revalidate'); @@ -1287,6 +1289,8 @@ describe(suiteName, function() { var layergroupTtl = 300; global.environment.varnish.layergroupTtl = layergroupTtl; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=' + layergroupTtl + ',must-revalidate');