From 8333b39928cce0ba3082cca641cc259447ec74fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 19 Mar 2018 19:16:18 +0100 Subject: [PATCH 01/14] Use res.body as placeholder of layergroup --- lib/cartodb/controllers/map.js | 40 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 52d71f93..9f15383f 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -280,7 +280,7 @@ function createLayergroup (mapBackend, userLimitsApi) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; next(); }); @@ -299,7 +299,7 @@ function instantiateLayergroup (mapBackend, userLimitsApi) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; const { mapconfigProvider } = res.locals; @@ -332,7 +332,7 @@ function incrementMapViewCount (metadataBackend) { function augmentLayergroupData () { return function augmentLayergroupDataMiddleware (req, res, next) { - const { layergroup } = res.locals; + const layergroup = res.body; // include in layergroup response the variables in serverMedata // those variables are useful to send to the client information @@ -345,7 +345,8 @@ function augmentLayergroupData () { function getAffectedTables (pgConnection, layergroupAffectedTables) { return function getAffectedTablesMiddleware (req, res, next) { - const { dbname, layergroup, user, mapconfig } = res.locals; + const { dbname, user, mapconfig } = res.locals; + const layergroup = res.body; pgConnection.getConnection(user, (err, connection) => { if (err) { @@ -403,7 +404,8 @@ function setLastModified () { function setLastUpdatedTimeToLayergroup () { return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { affectedTables, layergroup, analysesResults } = res.locals; + const { affectedTables, analysesResults } = res.locals; + const layergroup = res.body; var lastUpdateTime = affectedTables.getLastUpdatedAt(); @@ -443,7 +445,8 @@ function setCacheControl () { function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { - const { user, mapconfig, layergroup } = res.locals; + const { user, mapconfig } = res.locals; + const layergroup = res.body; pgConnection.getConnection(user, (err, connection) => { if (err) { @@ -469,7 +472,8 @@ function setLayerStats (pgConnection, statsBackend) { function setLayergroupIdHeader (templateMaps, useTemplateHash) { return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { layergroup, user, template } = res.locals; + const { user, template } = res.locals; + const layergroup = res.body; if (useTemplateHash) { var templateHash = templateMaps.fingerPrint(template).substring(0, 8); @@ -484,7 +488,8 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) { function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapconfig } = res.locals; + const layergroup = res.body; layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); @@ -494,7 +499,8 @@ function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, user, analysesResults = [] } = res.locals; + const { user, analysesResults = [] } = res.locals; + const layergroup = res.body; layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); @@ -504,7 +510,8 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { function setTurboCartoMetadataToLayergroup (layergroupMetadata) { return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapconfig, context } = res.locals; + const layergroup = res.body; layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context); @@ -514,7 +521,8 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) { function setAggregationMetadataToLayergroup (layergroupMetadata) { return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapconfig, context } = res.locals; + const layergroup = res.body; layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context); @@ -524,7 +532,8 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) { function setTilejsonMetadataToLayergroup (layergroupMetadata) { return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapconfig } = res.locals; + const layergroup = res.body; layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig); @@ -551,14 +560,13 @@ function setSurrogateKeyHeader (surrogateKeysCache) { function sendResponse () { return function sendResponseMiddleware (req, res) { req.profiler.done('res'); - const { layergroup } = res.locals; - res.status(200); + res.status(res.statusCode || 200); if (req.query && req.query.callback) { - res.jsonp(layergroup); + res.jsonp(res.body); } else { - res.json(layergroup); + res.json(res.body); } }; } From bb170ee208ea188a4d08791712c2cfdf35a79c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 19 Mar 2018 19:27:38 +0100 Subject: [PATCH 02/14] Please, jshint --- test/acceptance/ported/support/test_client.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/acceptance/ported/support/test_client.js b/test/acceptance/ported/support/test_client.js index f110079a..ab4576b6 100644 --- a/test/acceptance/ported/support/test_client.js +++ b/test/acceptance/ported/support/test_client.js @@ -456,17 +456,17 @@ function withLayergroup(layergroupConfig, options, callback) { const signerTpl = function ({ signer }) { return `${signer ? `:${signer}@` : ''}`; - } + }; const cacheTpl = function ({ cache_buster, cacheBuster }) { return `${cache_buster ? `:${cache_buster}` : `:${cacheBuster}`}`; - } + }; const urlTpl = function ({layergroupid, cache_buster = null, tile }) { const { signer, token , cacheBuster } = LayergroupToken.parse(layergroupid); - const base = '/database/windshaft_test/layergroup/' + const base = '/database/windshaft_test/layergroup/'; return `${base}${signerTpl({signer})}${token}${cacheTpl({cache_buster, cacheBuster})}${tile}`; - } + }; const finalUrl = urlTpl({ layergroupid, cache_buster: options.cache_buster, tile: layergroupUrl }); From 9211fa065b46acd306ff7a6d29dae67855a5dd95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 19 Mar 2018 19:48:14 +0100 Subject: [PATCH 03/14] Extract sendResponse middleware --- lib/cartodb/controllers/analyses.js | 13 +------------ lib/cartodb/controllers/layergroup.js | 19 +------------------ lib/cartodb/controllers/map.js | 15 +-------------- lib/cartodb/controllers/named_maps.js | 16 ++++------------ lib/cartodb/controllers/named_maps_admin.js | 10 +--------- lib/cartodb/middleware/send-response.js | 17 +++++++++++++++++ 6 files changed, 25 insertions(+), 65 deletions(-) create mode 100644 lib/cartodb/middleware/send-response.js diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 57e64a61..908ab3c6 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -7,6 +7,7 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const authorize = require('../middleware/authorize'); const dbConnSetup = require('../middleware/db-conn-setup'); +const sendResponse = require('../middleware/send-response'); function AnalysesController(pgConnection, authApi) { this.pgConnection = pgConnection; @@ -114,18 +115,6 @@ function setCacheControlHeader () { }; } -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - }; -} - function unauthorizedError () { return function unathorizedErrorMiddleware(err, req, res, next) { if (err.message.match(/permission\sdenied/)) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index f3976cb6..53d2ba7f 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -7,6 +7,7 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const sendResponse = require('../middleware/send-response'); const DataviewBackend = require('../backends/dataview'); const AnalysisStatusBackend = require('../backends/analysis-status'); const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); @@ -644,24 +645,6 @@ function incrementSuccessMetrics (statsClient) { }; } -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - - res.status(res.statusCode || 200); - - if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - } else { - res.send(res.body); - } - }; -} - function incrementErrorMetrics (statsClient) { return function incrementErrorMetricsMiddleware (err, req, res, next) { const formatStat = parseFormat(req.params.format); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 9f15383f..8022c7a2 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -12,6 +12,7 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const sendResponse = require('../middleware/send-response'); const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); @@ -557,20 +558,6 @@ function setSurrogateKeyHeader (surrogateKeysCache) { }; } -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - - res.status(res.statusCode || 200); - - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - }; -} - function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 6bff861c..95fe24b8 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -7,6 +7,7 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const sendResponse = require('../middleware/send-response'); const vectorError = require('../middleware/vector-error'); const DEFAULT_ZOOM_CENTER = { @@ -214,10 +215,11 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) function getTile ({ tileBackend, label }) { return function getTileMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { namedMapProvider, format } = res.locals; tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => { req.profiler.add(stats); + req.profiler.done('render-' + format); if (err) { err.label = label; @@ -371,6 +373,7 @@ function getImage({ previewBackend, label }) { previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => { req.profiler.add(stats); + req.profiler.done('render-' + format); if (err) { err.label = label; @@ -514,14 +517,3 @@ function setContentTypeHeader () { next(); }; } - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - const { format } = res.locals; - - req.profiler.done('render-' + format); - - res.status(200); - res.send(res.body); - }; -} diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index c2e7725e..f70af2cf 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -3,6 +3,7 @@ const cors = require('../middleware/cors'); const user = require('../middleware/user'); const locals = require('../middleware/locals'); const credentials = require('../middleware/credentials'); +const sendResponse = require('../middleware/send-response'); /** * @param {AuthApi} authApi @@ -216,12 +217,3 @@ function listTemplates ({ templateMaps }) { }); }; } - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(res.statusCode || 200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method](res.body); - }; -} diff --git a/lib/cartodb/middleware/send-response.js b/lib/cartodb/middleware/send-response.js new file mode 100644 index 00000000..469cf0a7 --- /dev/null +++ b/lib/cartodb/middleware/send-response.js @@ -0,0 +1,17 @@ +module.exports = function sendResponse () { + return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); + + res.status(res.statusCode || 200); + + if (Buffer.isBuffer(res.body)) { + return res.send(res.body); + } + + if (req.query.callback) { + return res.jsonp(res.body); + } + + res.json(res.body); + }; +}; From 325bdfe92ffe51b2a733c1638942f66a9e703b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 20 Mar 2018 09:34:06 +0100 Subject: [PATCH 04/14] Move middleware --- lib/cartodb/controllers/map.js | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 8022c7a2..61d42ed4 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -89,6 +89,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal augmentLayergroupData(), getAffectedTables(this.pgConnection, this.layergroupAffectedTables), setCacheChannel(), + setSurrogateKeyHeader(this.surrogateKeysCache), setLastModified(), setLastUpdatedTimeToLayergroup(), setCacheControl(), @@ -99,7 +100,6 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal setTurboCartoMetadataToLayergroup(this.layergroupMetadata), setAggregationMetadataToLayergroup(this.layergroupMetadata), setTilejsonMetadataToLayergroup(this.layergroupMetadata), - setSurrogateKeyHeader(this.surrogateKeysCache), sendResponse(), augmentError({ label, addContext }) ]; @@ -393,6 +393,22 @@ function setCacheChannel () { }; } +function setSurrogateKeyHeader (surrogateKeysCache) { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { affectedTables, user, templateName } = res.locals; + + if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { + surrogateKeysCache.tag(res, affectedTables); + } + + if (templateName) { + surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); + } + + next(); + }; +} + function setLastModified () { return function setLastModifiedMiddleware (req, res, next) { if (req.method === 'GET') { @@ -542,22 +558,6 @@ function setTilejsonMetadataToLayergroup (layergroupMetadata) { }; } -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { affectedTables, user, templateName } = res.locals; - - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTables); - } - - if (templateName) { - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } - - next(); - }; -} - function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; From 9fd2519c12b5982c06d3f37d472f99aafa06fbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 20 Mar 2018 09:34:50 +0100 Subject: [PATCH 05/14] Rename middleware --- lib/cartodb/controllers/map.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 61d42ed4..d9834a05 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -88,7 +88,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal incrementMapViewCount(this.metadataBackend), augmentLayergroupData(), getAffectedTables(this.pgConnection, this.layergroupAffectedTables), - setCacheChannel(), + setCacheChannelHeader(), setSurrogateKeyHeader(this.surrogateKeysCache), setLastModified(), setLastUpdatedTimeToLayergroup(), @@ -381,8 +381,8 @@ function getAffectedTables (pgConnection, layergroupAffectedTables) { }; } -function setCacheChannel () { - return function setCacheChannelMiddleware (req, res, next) { +function setCacheChannelHeader () { + return function setCacheChannelHeaderMiddleware (req, res, next) { const { affectedTables } = res.locals; if (req.method === 'GET') { From a142620b70311f2654cff6dd5a10284e35c5013b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Mar 2018 14:11:54 +0100 Subject: [PATCH 06/14] Make generic middlewares to calculate surrogate key and cache channel headers: - In controllers: all reference to map config are now camelized, for instance: mapconfig -> mapConfig or mapconfigProvider -> mapConfigProvider - In controllers: all map config providers created in req/res cycle are saved into `res.locals` and `mapConfigProvider` as key. - In map-config-providers: all of them implement `.getAffectedTables()`, in order to calculate the tables involved for a given map-config. For that, `pgConnection` and `affectedTablesCache` are injected as constructor argument. - Named Map Provider: rename references from `affectedTablesAndLastUpdate` to `affectedTables`. - Named Map Provider Cache: In order to create new named map providers, needs affectedTablesCache. - Extract locals middlewares (surrogate-key and cache-channel) from controllers and create an unified version of them. - Extract last-modified middleware from named maps controller (draft). --- lib/cartodb/cache/named_map_provider_cache.js | 11 +- lib/cartodb/controllers/layergroup.js | 252 ++++++++---------- lib/cartodb/controllers/map.js | 213 +++++++-------- lib/cartodb/controllers/named_maps.js | 122 ++------- .../middleware/cache-channel-header.js | 24 ++ .../middleware/last-modified-header.js | 40 +++ .../middleware/surrogate-key-header.js | 31 +++ .../provider/create-layergroup-provider.js | 64 ++++- .../mapconfig/provider/map-store-provider.js | 82 +++++- .../mapconfig/provider/named-map-provider.js | 67 ++++- lib/cartodb/server.js | 3 +- 11 files changed, 538 insertions(+), 371 deletions(-) create mode 100644 lib/cartodb/middleware/cache-channel-header.js create mode 100644 lib/cartodb/middleware/last-modified-header.js create mode 100644 lib/cartodb/middleware/surrogate-key-header.js diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index ebafbbac..e0850822 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -6,12 +6,20 @@ var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.mapConfigAdapter = mapConfigAdapter; + this.affectedTablesCache = affectedTablesCache; this.providerCache = new LruCache({ max: 2000 }); } @@ -30,6 +38,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.metadataBackend, this.userLimitsApi, this.mapConfigAdapter, + this.affectedTablesCache, user, templateId, config, diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 29adf431..b7b6803a 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -9,11 +9,12 @@ const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); const sendResponse = require('../middleware/send-response'); const DataviewBackend = require('../backends/dataview'); const AnalysisStatusBackend = require('../backends/analysis-status'); const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); -const QueryTables = require('cartodb-query-tables'); const SUPPORTED_FORMATS = { grid_json: true, json_torque: true, @@ -44,7 +45,7 @@ function LayergroupController( attributesBackend, surrogateKeysCache, userLimitsApi, - layergroupAffectedTables, + layergroupAffectedTablesCache, analysisBackend, authApi ) { @@ -55,7 +56,7 @@ function LayergroupController( this.attributesBackend = attributesBackend; this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.dataviewBackend = new DataviewBackend(analysisBackend); this.analysisStatusBackend = new AnalysisStatusBackend(); @@ -65,10 +66,10 @@ function LayergroupController( module.exports = LayergroupController; LayergroupController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath } = app; + const { base_url_mapconfig: mapConfigBasePath } = app; app.get( - `${mapconfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, + `${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, cors(), cleanUpQueryParams(), locals(), @@ -78,13 +79,17 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getTile(this.tileBackend, 'map_tile'), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -93,7 +98,7 @@ LayergroupController.prototype.register = function(app) { ); app.get( - `${mapconfigBasePath}/:token/:z/:x/:y.:format`, + `${mapConfigBasePath}/:token/:z/:x/:y.:format`, cors(), cleanUpQueryParams(), locals(), @@ -103,13 +108,17 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getTile(this.tileBackend, 'map_tile'), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -118,7 +127,7 @@ LayergroupController.prototype.register = function(app) { ); app.get( - `${mapconfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, + `${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, distinguishLayergroupFromStaticRoute(), cors(), cleanUpQueryParams(), @@ -129,13 +138,17 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getTile(this.tileBackend, 'maplayer_tile'), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -144,7 +157,7 @@ LayergroupController.prototype.register = function(app) { ); app.get( - `${mapconfigBasePath}/:token/:layer/attributes/:fid`, + `${mapConfigBasePath}/:token/:layer/attributes/:fid`, cors(), cleanUpQueryParams(), locals(), @@ -154,20 +167,24 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getFeatureAttributes(this.attributesBackend), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), sendResponse() ); const forcedFormat = 'png'; app.get( - `${mapconfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, + `${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, cors(), cleanUpQueryParams(['layer']), locals(), @@ -177,18 +194,23 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), getPreviewImageByCenter(this.previewBackend), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), sendResponse() ); app.get( - `${mapconfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, + `${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, cors(), cleanUpQueryParams(['layer']), locals(), @@ -198,13 +220,18 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), getPreviewImageByBoundingBox(this.previewBackend), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), sendResponse() ); @@ -227,7 +254,7 @@ LayergroupController.prototype.register = function(app) { ]; app.get( - `${mapconfigBasePath}/:token/dataview/:dataviewName`, + `${mapConfigBasePath}/:token/dataview/:dataviewName`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -237,18 +264,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getDataview(this.dataviewBackend), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/:layer/widget/:dataviewName`, + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -258,18 +289,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), getDataview(this.dataviewBackend), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/dataview/:dataviewName/search`, + `${mapConfigBasePath}/:token/dataview/:dataviewName/search`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -279,18 +314,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), dataviewSearch(this.dataviewBackend), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/:layer/widget/:dataviewName/search`, + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`, cors(), cleanUpQueryParams(allowedDataviewQueryParams), locals(), @@ -300,18 +339,22 @@ LayergroupController.prototype.register = function(app) { credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), dataviewSearch(this.dataviewBackend), setCacheControlHeader(), setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), sendResponse() ); app.get( - `${mapconfigBasePath}/:token/analysis/node/:nodeId`, + `${mapConfigBasePath}/:token/analysis/node/:nodeId`, cors(), cleanUpQueryParams(), locals(), @@ -367,7 +410,13 @@ function getRequestParams(locals) { return params; } -function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) { +function createMapStoreMapConfigProvider ( + mapStore, + userLimitsApi, + pgConnection, + affectedTablesCache, + forcedFormat = null +) { return function createMapStoreMapConfigProviderMiddleware (req, res, next) { const { user } = res.locals; @@ -378,7 +427,14 @@ function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat params.layer = params.layer || 'all'; } - res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params); + res.locals.mapConfigProvider = new MapStoreMapConfigProvider( + mapStore, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + params + ); next(); }; @@ -575,88 +631,6 @@ function setCacheControlHeader () { }; } -function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) { - return function getAffectedTablesMiddleware (req, res, next) { - const { user, dbname, token } = res.locals; - - if (layergroupAffectedTables.hasAffectedTables(dbname, token)) { - res.locals.affectedTables = layergroupAffectedTables.get(dbname, token); - return next(); - } - - mapStore.load(token, (err, mapconfig) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - const queries = []; - mapconfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - queries.push(`SELECT * FROM ${table} LIMIT 0`); - }); - } - }); - - const sql = queries.length ? queries.join(';') : null; - - if (!sql) { - global.logger.warn('ERROR generating cache channel:' + - ' this request doesn\'t need an X-Cache-Channel generated'); - return next(); - } - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - global.logger.warn('ERROR generating cache channel: ', err); - return next(); - } - - // feed affected tables cache so it can be reused from, for instance, map controller - layergroupAffectedTables.set(dbname, token, affectedTables); - - res.locals.affectedTables = affectedTables; - - next(); - }); - }); - }); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - surrogateKeysCache.tag(res, affectedTables); - } - - next(); - }; -} - function incrementSuccessMetrics (statsClient) { return function incrementSuccessMetricsMiddleware (req, res, next) { const formatStat = parseFormat(req.params.format); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 413f023d..ea8b309a 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,7 +2,6 @@ const _ = require('underscore'); const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const QueryTables = require('cartodb-query-tables'); const ResourceLocator = require('../models/resource-locator'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); @@ -12,8 +11,9 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); const sendResponse = require('../middleware/send-response'); -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../utils/layergroup-metadata'); @@ -64,15 +64,15 @@ function MapController ( module.exports = MapController; MapController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app; + const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app; app.get( - `${mapconfigBasePath}`, + `${mapConfigBasePath}`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) ); app.post( - `${mapconfigBasePath}`, + `${mapConfigBasePath}`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) ); @@ -88,7 +88,7 @@ MapController.prototype.register = function(app) { this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) ); - app.options(app.base_url_mapconfig, cors('Content-Type')); + app.options(`${mapConfigBasePath}`, cors('Content-Type')); }; MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { @@ -113,9 +113,8 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us this.getCreateMapMiddlewares(useTemplate), incrementMapViewCount(this.metadataBackend), augmentLayergroupData(), - getAffectedTables(this.pgConnection, this.layergroupAffectedTables), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), setLastModified(), setLastUpdatedTimeToLayergroup(), setCacheControl(), @@ -140,16 +139,27 @@ MapController.prototype.getCreateMapMiddlewares = function (useTemplate) { this.pgConnection, this.metadataBackend, this.userLimitsApi, - this.mapConfigAdapter + this.mapConfigAdapter, + this.layergroupAffectedTables ), - instantiateLayergroup(this.mapBackend, this.userLimitsApi) + instantiateLayergroup( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; } return [ checkCreateLayergroup(), prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup (this.mapBackend, this.userLimitsApi) + createLayergroup ( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; }; @@ -220,17 +230,25 @@ function checkCreateLayergroup () { }; } -function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function getTemplate ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { return function getTemplateMiddleware (req, res, next) { const templateParams = req.body; const { user } = res.locals; - const mapconfigProvider = new NamedMapMapConfigProvider( + const mapConfigProvider = new NamedMapMapConfigProvider( templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, + affectedTablesCache, user, req.params.template_id, templateParams, @@ -238,15 +256,15 @@ function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi res.locals ); - mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { req.profiler.done('named.getMapConfig'); if (err) { return next(err); } - res.locals.mapconfig = mapconfig; + res.locals.mapConfig = mapConfig; res.locals.rendererParams = rendererParams; - res.locals.mapconfigProvider = mapconfigProvider; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); @@ -289,38 +307,51 @@ function prepareAdapterMapConfig (mapConfigAdapter) { }; } -function createLayergroup (mapBackend, userLimitsApi) { +function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function createLayergroupMiddleware (req, res, next) { const requestMapConfig = req.body; const { context, user } = res.locals; const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapconfig = new MapConfig(requestMapConfig, datasource); - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals); + const mapConfig = new MapConfig(requestMapConfig, datasource); + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + res.locals + ); - res.locals.mapconfig = mapconfig; + res.locals.mapConfig = mapConfig; res.locals.analysesResults = context.analysesResults; - mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + mapBackend.createLayergroup(mapConfig, res.locals, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); } res.body = layergroup; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); }; } -function instantiateLayergroup (mapBackend, userLimitsApi) { +function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapconfig, rendererParams } = res.locals; - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams); + const { user, mapConfig, rendererParams } = res.locals; + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + rendererParams + ); - mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); @@ -328,12 +359,11 @@ function instantiateLayergroup (mapBackend, userLimitsApi) { res.body = layergroup; - const { mapconfigProvider } = res.locals; + const { mapConfigProvider } = res.locals; - res.locals.analysesResults = mapconfigProvider.analysesResults; - res.locals.template = mapconfigProvider.template; - res.locals.templateName = mapconfigProvider.getTemplateName(); - res.locals.context = mapconfigProvider.context; + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; next(); }); @@ -342,10 +372,10 @@ function instantiateLayergroup (mapBackend, userLimitsApi) { function incrementMapViewCount (metadataBackend) { return function incrementMapViewCountMiddleware(req, res, next) { - const { mapconfig, user } = res.locals; + const { mapConfig, user } = res.locals; // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { req.profiler.done('incMapviewCount'); if (err) { @@ -370,71 +400,6 @@ function augmentLayergroupData () { }; } -function getAffectedTables (pgConnection, layergroupAffectedTables) { - return function getAffectedTablesMiddleware (req, res, next) { - const { dbname, user, mapconfig } = res.locals; - const layergroup = res.body; - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - const sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); - - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - return next(err); - } - - // feed affected tables cache so it can be reused from, for instance, layergroup controller - layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); - - res.locals.affectedTables = affectedTables; - - next(); - }); - }); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (req.method === 'GET') { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { affectedTables, user, templateName } = res.locals; - - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTables); - } - - if (templateName) { - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } - - next(); - }; -} - function setLastModified () { return function setLastModifiedMiddleware (req, res, next) { if (req.method === 'GET') { @@ -447,18 +412,28 @@ function setLastModified () { function setLastUpdatedTimeToLayergroup () { return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { affectedTables, analysesResults } = res.locals; + const { mapConfigProvider, analysesResults } = res.locals; const layergroup = res.body; - var lastUpdateTime = affectedTables.getLastUpdatedAt(); + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + return next(err); + } - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + if (!affectedTables) { + return next(); + } - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + var lastUpdateTime = affectedTables.getLastUpdatedAt(); - next(); + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + next(); + }); }; } @@ -488,7 +463,7 @@ function setCacheControl () { function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { - const { user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; const layergroup = res.body; pgConnection.getConnection(user, (err, connection) => { @@ -496,7 +471,7 @@ function setLayerStats (pgConnection, statsBackend) { return next(err); } - statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { if (err) { return next(err); } @@ -531,10 +506,10 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) { function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; const layergroup = res.body; - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); next(); }; @@ -553,10 +528,10 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { function setTurboCartoMetadataToLayergroup (layergroupMetadata) { return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; const layergroup = res.body; - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -564,10 +539,10 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) { function setAggregationMetadataToLayergroup (layergroupMetadata) { return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; const layergroup = res.body; - layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -575,10 +550,10 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) { function setTilejsonMetadataToLayergroup (layergroupMetadata) { return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; const layergroup = res.body; - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig); + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); next(); }; @@ -589,10 +564,10 @@ function augmentError (options) { return function augmentErrorMiddleware (err, req, res, next) { req.profiler.done('error'); - const { mapconfig } = res.locals; + const { mapConfig } = res.locals; if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; } err.label = label; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 74c628a5..c9d67c95 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -1,4 +1,3 @@ -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); const locals = require('../middleware/locals'); @@ -7,6 +6,9 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); const sendResponse = require('../middleware/send-response'); const vectorError = require('../middleware/vector-error'); const rateLimit = require('../middleware/rate-limit'); @@ -28,8 +30,8 @@ function getRequestParams(locals) { const params = Object.assign({}, locals); delete params.template; - delete params.affectedTablesAndLastUpdate; - delete params.namedMapProvider; + delete params.affectedTables; + delete params.mapConfigProvider; delete params.allowedQueryParams; return params; @@ -77,14 +79,13 @@ NamedMapsController.prototype.register = function(app) { namedMapProviderCache: this.namedMapProviderCache, label: 'NAMED_MAP_TILE' }), - getAffectedTables(), getTile({ tileBackend: this.tileBackend, label: 'NAMED_MAP_TILE' }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), setCacheControlHeader(), setContentTypeHeader(), sendResponse(), @@ -106,7 +107,6 @@ NamedMapsController.prototype.register = function(app) { namedMapProviderCache: this.namedMapProviderCache, label: 'STATIC_VIZ_MAP', forcedFormat: 'png' }), - getAffectedTables(), getTemplate({ label: 'STATIC_VIZ_MAP' }), prepareLayerFilterFromPreviewLayers({ namedMapProviderCache: this.namedMapProviderCache, @@ -115,9 +115,9 @@ NamedMapsController.prototype.register = function(app) { getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), incrementMapViews({ metadataBackend: this.metadataBackend }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + cacheChannelHeader(), + lastModifiedHeader(), setCacheControlHeader(), setContentTypeHeader(), sendResponse() @@ -143,25 +143,7 @@ function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = nul return next(err); } - res.locals.namedMapProvider = namedMapProvider; - - next(); - }); - }; -} - -function getAffectedTables () { - return function getAffectedTables (req, res, next) { - const { namedMapProvider } = res.locals; - - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { - req.profiler.done('affectedTables'); - - if (err) { - return next(err); - } - - res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate; + res.locals.mapConfigProvider = namedMapProvider; next(); }); @@ -170,9 +152,9 @@ function getAffectedTables () { function getTemplate ({ label }) { return function getTemplateMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { mapConfigProvider } = res.locals; - namedMapProvider.getTemplate((err, template) => { + mapConfigProvider.getTemplate((err, template) => { if (err) { err.label = label; return next(err); @@ -220,7 +202,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) return next(err); } - res.locals.namedMapProvider = provider; + res.locals.mapConfigProvider = provider; next(); }); @@ -229,9 +211,9 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) function getTile ({ tileBackend, label }) { return function getTileMiddleware (req, res, next) { - const { namedMapProvider, format } = res.locals; + const { mapConfigProvider, format } = res.locals; - tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => { + tileBackend.getTile(mapConfigProvider, req.params, (err, tile, headers, stats) => { req.profiler.add(stats); req.profiler.done('render-' + format); @@ -253,7 +235,7 @@ function getTile ({ tileBackend, label }) { function getStaticImageOptions ({ tablesExtentApi }) { return function getStaticImageOptionsMiddleware(req, res, next) { - const { user, namedMapProvider, template } = res.locals; + const { user, mapConfigProvider, template } = res.locals; const imageOpts = getImageOptions(res.locals, template); @@ -264,18 +246,18 @@ function getStaticImageOptions ({ tablesExtentApi }) { res.locals.imageOpts = DEFAULT_ZOOM_CENTER; - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { + mapConfigProvider.getAffectedTables((err, affectedTables) => { if (err) { return next(); } - var affectedTables = affectedTablesAndLastUpdate.tables || []; + var tables = affectedTables.tables || []; - if (affectedTables.length === 0) { + if (tables.length === 0) { return next(); } - tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => { + tablesExtentApi.getBounds(user, tables, (err, bounds) => { if (err) { return next(); } @@ -355,7 +337,7 @@ function getImageOptionsFromBoundingBox (bbox = '') { function getImage({ previewBackend, label }) { return function getImageMiddleware (req, res, next) { - const { imageOpts, namedMapProvider } = res.locals; + const { imageOpts, mapConfigProvider } = res.locals; const { zoom, center, bounds } = imageOpts; let { width, height } = req.params; @@ -366,7 +348,7 @@ function getImage({ previewBackend, label }) { const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; if (zoom !== undefined && center) { - return previewBackend.getImage(namedMapProvider, format, width, height, zoom, center, + return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center, (err, image, headers, stats) => { req.profiler.add(stats); @@ -385,7 +367,7 @@ function getImage({ previewBackend, label }) { }); } - previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => { + previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => { req.profiler.add(stats); req.profiler.done('render-' + format); @@ -411,9 +393,9 @@ function incrementMapViewsError (ctx) { function incrementMapViews ({ metadataBackend }) { return function incrementMapViewsMiddleware(req, res, next) { - const { user, namedMapProvider } = res.locals; + const { user, mapConfigProvider } = res.locals; - namedMapProvider.getMapConfig((err, mapConfig) => { + mapConfigProvider.getMapConfig((err, mapConfig) => { if (err) { global.logger.log(incrementMapViewsError({ user, err })); return next(); @@ -462,59 +444,13 @@ function templateBounds(view) { return false; } -function setSurrogateKeyHeader ({ surrogateKeysCache }) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals; - - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName())); - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - if (affectedTablesAndLastUpdate.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTablesAndLastUpdate); - } - } - - next(); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel()); - } - - next(); - }; -} - -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - var lastModifiedDate; - if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) { - lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt()); - } else { - lastModifiedDate = new Date(); - } - - res.set('Last-Modified', lastModifiedDate.toUTCString()); - } - - next(); - }; - } - function setCacheControlHeader () { return function setCacheControlHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; + const { affectedTables } = res.locals; res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { + if (!affectedTables || !!affectedTables.tables) { // we increase cache control as we can invalidate it res.set('Cache-Control', 'public,max-age=31536000'); } diff --git a/lib/cartodb/middleware/cache-channel-header.js b/lib/cartodb/middleware/cache-channel-header.js new file mode 100644 index 00000000..d6bf394b --- /dev/null +++ b/lib/cartodb/middleware/cache-channel-header.js @@ -0,0 +1,24 @@ +module.exports = function setCacheChannelHeader () { + return function setCacheChannelHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider } = res.locals; + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Cache Channel Header:', err); + return next(); + } + + if (!affectedTables) { + return next(); + } + + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/last-modified-header.js b/lib/cartodb/middleware/last-modified-header.js new file mode 100644 index 00000000..51211661 --- /dev/null +++ b/lib/cartodb/middleware/last-modified-header.js @@ -0,0 +1,40 @@ +module.exports = function setLastModifiedHeader () { + return function setLastModifiedHeaderMiddleware(req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider, cache_buster } = res.locals; + + if (cache_buster) { + const cacheBuster = parseInt(cache_buster, 10); + + if (Number.isFinite(cacheBuster)) { + res.set('Last-Modified', new Date(cacheBuster).toUTCString()); + } + + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Last Modified Header:', err); + return next(); + } + + if (!affectedTables) { + return next(); + } + + const lastUpdatedAt = affectedTables.getLastUpdatedAt(); + + const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? + new Date(lastUpdatedAt) : + new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/surrogate-key-header.js b/lib/cartodb/middleware/surrogate-key-header.js new file mode 100644 index 00000000..51cec1c1 --- /dev/null +++ b/lib/cartodb/middleware/surrogate-key-header.js @@ -0,0 +1,31 @@ +const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); + +module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { user, mapConfigProvider } = res.locals; + + if (mapConfigProvider instanceof NamedMapMapConfigProvider) { + surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName())); + } + + if (req.method !== 'GET') { + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Surrogate Key Header:', err); + return next(); + } + + if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) { + return next(); + } + + surrogateKeysCache.tag(res, affectedTables); + + next(); + }); + }; +}; diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 340073b5..0eef42f8 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -2,6 +2,7 @@ var assert = require('assert'); var step = require('step'); var MapStoreMapConfigProvider = require('./map-store-provider'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapConfig} mapConfig @@ -11,10 +12,13 @@ var MapStoreMapConfigProvider = require('./map-store-provider'); * @constructor * @type {CreateLayergroupMapConfigProvider} */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) { + +function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapConfig = mapConfig; this.user = user; this.userLimitsApi = userLimitsApi; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.params = params; this.cacheBuster = params.cache_buster || 0; } @@ -46,3 +50,61 @@ CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigPr CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter; CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey; + +CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) { + var self = this; + + const { dbname } = self.params; + const token = self.mapConfig.id(); + + if (self.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = self.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + step( + function getSql() { + const queries = []; + + self.mapConfig.getLayers().forEach(function(layer) { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + return sql; + }, + function getAffectedTables(err, sql) { + assert.ifError(err); + + step( + function getConnection() { + self.pgConnection.getConnection(self.user, this); + }, + function getAffectedTables(err, connection) { + assert.ifError(err); + QueryTables.getAffectedTablesFromQuery(connection, sql, this); + }, + this + ); + }, + function finish(err, affectedTables) { + if (err) { + return callback(err); + } + + self.affectedTablesCache.set(dbname, token, affectedTables); + + return callback(null, affectedTables); + } + ); +}; diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index 177322d4..2c94de92 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var dot = require('dot'); var step = require('step'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapStore} mapStore @@ -11,20 +12,30 @@ var step = require('step'); * @constructor * @type {MapStoreMapConfigProvider} */ -function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) { +function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapStore = mapStore; this.user = user; this.userLimitsApi = userLimitsApi; - this.params = params; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.token = params.token; this.cacheBuster = params.cache_buster || 0; + this.mapConfig = null; + this.params = params; + this.context = null; } module.exports = MapStoreMapConfigProvider; MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig !== null) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -39,6 +50,8 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { self.mapStore.load(self.token, this); }, function finish(err, mapConfig) { + self.mapConfig = mapConfig; + self.context = context; return callback(err, mapConfig, self.params, context); } ); @@ -74,4 +87,67 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) { scale_factor: 1 }); return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); -}; \ No newline at end of file +}; + +MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) { + var self = this; + + const { dbname, token } = self.params; + + if (self.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = self.affectedTablesCache.get(dbname, token); + + return callback(null, affectedTables); + } + + step( + function getMapConfig() { + self.getMapConfig(this); + }, + function getSql(err, mapConfig) { + assert.ifError(err); + + const queries = []; + + mapConfig.getLayers().forEach(function(layer) { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + return sql; + }, + function getAffectedTables(err, sql) { + assert.ifError(err); + + step( + function getConnection() { + self.pgConnection.getConnection(self.user, this); + }, + function getAffectedTables(err, connection) { + assert.ifError(err); + QueryTables.getAffectedTablesFromQuery(connection, sql, this); + }, + this + ); + }, + function finish(err, affectedTables) { + if (err) { + return callback(err); + } + + self.affectedTablesCache.set(dbname, token, affectedTables); + + return callback(err, affectedTables); + } + ); +}; diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 9b085dbc..0fa003e2 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -11,8 +11,19 @@ var QueryTables = require('cartodb-query-tables'); * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, - owner, templateId, config, authToken, params) { +function NamedMapMapConfigProvider( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache, + owner, + templateId, + config, + authToken, + params +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; @@ -30,7 +41,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, // use template after call to mapConfig this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTablesCache = affectedTablesCache; // providing this.err = null; @@ -189,7 +200,7 @@ NamedMapMapConfigProvider.prototype.getCacheBuster = function() { NamedMapMapConfigProvider.prototype.reset = function() { this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTables = null; this.err = null; this.mapConfig = null; @@ -251,12 +262,11 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() { return this.templateName; }; -NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) { +NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) { var self = this; - if (this.affectedTablesAndLastUpdate !== null) { - return callback(null, this.affectedTablesAndLastUpdate); - } + let dbname = null; + let token = null; step( function getMapConfig() { @@ -264,9 +274,33 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi }, function getSql(err, mapConfig) { assert.ifError(err); - return mapConfig.getLayers().map(function(layer) { - return layer.options.sql; - }).join(';'); + + dbname = self.rendererParams; + token = mapConfig.id(); + + if (self.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = self.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + const queries = []; + + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + return sql; }, function getAffectedTables(err, sql) { assert.ifError(err); @@ -281,9 +315,14 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi this ); }, - function finish(err, result) { - self.affectedTablesAndLastUpdate = result; - return callback(err, result); + function finish(err, affectedTables) { + if (err) { + return callback(err); + } + + self.affectedTablesCache.set(dbname, token, affectedTables); + + return callback(err, affectedTables); } ); }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7f7e2712..8535783d 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -200,7 +200,8 @@ module.exports = function(serverOptions) { pgConnection, metadataBackend, userLimitsApi, - mapConfigAdapter + mapConfigAdapter, + layergroupAffectedTablesCache ); ['update', 'delete'].forEach(function(eventType) { From d022a1fa5e15de958c185c89974e0f23e6da1409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Mar 2018 14:43:00 +0100 Subject: [PATCH 07/14] Extract last-modified header middlleware --- lib/cartodb/controllers/layergroup.js | 35 ++++++------------- lib/cartodb/controllers/map.js | 13 ++----- .../middleware/last-modified-header.js | 21 ++++++----- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b7b6803a..9da86ff4 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -11,6 +11,7 @@ const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const cacheChannelHeader = require('../middleware/cache-channel-header'); const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); const sendResponse = require('../middleware/send-response'); const DataviewBackend = require('../backends/dataview'); const AnalysisStatusBackend = require('../backends/analysis-status'); @@ -87,9 +88,9 @@ LayergroupController.prototype.register = function(app) { ), getTile(this.tileBackend, 'map_tile'), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -116,9 +117,9 @@ LayergroupController.prototype.register = function(app) { ), getTile(this.tileBackend, 'map_tile'), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -146,9 +147,9 @@ LayergroupController.prototype.register = function(app) { ), getTile(this.tileBackend, 'maplayer_tile'), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), incrementSuccessMetrics(global.statsClient), sendResponse(), incrementErrorMetrics(global.statsClient), @@ -175,9 +176,9 @@ LayergroupController.prototype.register = function(app) { ), getFeatureAttributes(this.attributesBackend), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -203,9 +204,9 @@ LayergroupController.prototype.register = function(app) { ), getPreviewImageByCenter(this.previewBackend), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -229,9 +230,9 @@ LayergroupController.prototype.register = function(app) { ), getPreviewImageByBoundingBox(this.previewBackend), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -272,9 +273,9 @@ LayergroupController.prototype.register = function(app) { ), getDataview(this.dataviewBackend), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -297,9 +298,9 @@ LayergroupController.prototype.register = function(app) { ), getDataview(this.dataviewBackend), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -322,9 +323,9 @@ LayergroupController.prototype.register = function(app) { ), dataviewSearch(this.dataviewBackend), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -347,9 +348,9 @@ LayergroupController.prototype.register = function(app) { ), dataviewSearch(this.dataviewBackend), setCacheControlHeader(), - setLastModifiedHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); @@ -609,20 +610,6 @@ function getPreviewImageByBoundingBox (previewBackend) { }; } -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware (req, res, next) { - let { cache_buster: cacheBuster } = res.locals; - - cacheBuster = parseInt(cacheBuster, 10); - - const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date(); - - res.set('Last-Modified', lastUpdated.toUTCString()); - - next(); - }; -} - function setCacheControlHeader () { return function setCacheControlHeaderMiddleware (req, res, next) { res.set('Cache-Control', 'public,max-age=31536000'); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ea8b309a..670ed9d7 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -13,6 +13,7 @@ const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); const cacheChannelHeader = require('../middleware/cache-channel-header'); const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); const sendResponse = require('../middleware/send-response'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); @@ -115,7 +116,7 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us augmentLayergroupData(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setLastModified(), + lastModifiedHeader({ now: true }), setLastUpdatedTimeToLayergroup(), setCacheControl(), setLayerStats(this.pgConnection, this.statsBackend), @@ -400,16 +401,6 @@ function augmentLayergroupData () { }; } -function setLastModified () { - return function setLastModifiedMiddleware (req, res, next) { - if (req.method === 'GET') { - res.set('Last-Modified', (new Date()).toUTCString()); - } - - next(); - }; -} - function setLastUpdatedTimeToLayergroup () { return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { const { mapConfigProvider, analysesResults } = res.locals; diff --git a/lib/cartodb/middleware/last-modified-header.js b/lib/cartodb/middleware/last-modified-header.js index 51211661..18c0d961 100644 --- a/lib/cartodb/middleware/last-modified-header.js +++ b/lib/cartodb/middleware/last-modified-header.js @@ -1,4 +1,4 @@ -module.exports = function setLastModifiedHeader () { +module.exports = function setLastModifiedHeader ({ now = false } = {}) { return function setLastModifiedHeaderMiddleware(req, res, next) { if (req.method !== 'GET') { return next(); @@ -8,10 +8,16 @@ module.exports = function setLastModifiedHeader () { if (cache_buster) { const cacheBuster = parseInt(cache_buster, 10); + const lastModifiedDate = Number.isFinite(cacheBuster) ? new Date(cacheBuster) : new Date(); - if (Number.isFinite(cacheBuster)) { - res.set('Last-Modified', new Date(cacheBuster).toUTCString()); - } + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + return next(); + } + + // REVIEW: to keep 100% compatibility with maps controller + if (now) { + res.set('Last-Modified', new Date().toUTCString()); return next(); } @@ -23,14 +29,13 @@ module.exports = function setLastModifiedHeader () { } if (!affectedTables) { + res.set('Last-Modified', new Date().toUTCString()); + return next(); } const lastUpdatedAt = affectedTables.getLastUpdatedAt(); - - const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? - new Date(lastUpdatedAt) : - new Date(); + const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? new Date(lastUpdatedAt) : new Date(); res.set('Last-Modified', lastModifiedDate.toUTCString()); From 72c4a7abd6fc9ee22514b1ff0333b6cff58d2290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Mar 2018 16:38:37 +0100 Subject: [PATCH 08/14] Extract cache control header middleware --- lib/cartodb/controllers/analyses.js | 10 ++----- lib/cartodb/controllers/layergroup.js | 29 +++++++------------ lib/cartodb/controllers/map.js | 3 +- lib/cartodb/controllers/named_maps.js | 23 +++------------ .../middleware/cache-control-header.js | 17 +++++++++++ test/acceptance/multilayer.js | 4 +++ 6 files changed, 40 insertions(+), 46 deletions(-) create mode 100644 lib/cartodb/middleware/cache-control-header.js diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index a9eee758..f632b394 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -9,6 +9,7 @@ const authorize = require('../middleware/authorize'); const dbConnSetup = require('../middleware/db-conn-setup'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheControlHeader = require('../middleware/cache-control-header'); const sendResponse = require('../middleware/send-response'); function AnalysesController(pgConnection, authApi, userLimitsApi) { @@ -37,7 +38,7 @@ AnalysesController.prototype.register = function (app) { getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), prepareResponse(), - setCacheControlHeader(), + cacheControlHeader({ ttl: 10, revalidate: true }), sendResponse(), unauthorizedError() ); @@ -112,13 +113,6 @@ function prepareResponse () { }; } -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - next(); - }; -} - function unauthorizedError () { return function unathorizedErrorMiddleware(err, req, res, next) { if (err.message.match(/permission\sdenied/)) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 9da86ff4..a77a7168 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -9,6 +9,7 @@ const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheControlHeader = require('../middleware/cache-control-header'); const cacheChannelHeader = require('../middleware/cache-channel-header'); const surrogateKeyHeader = require('../middleware/surrogate-key-header'); const lastModifiedHeader = require('../middleware/last-modified-header'); @@ -87,7 +88,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -116,7 +117,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -146,7 +147,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), getTile(this.tileBackend, 'maplayer_tile'), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -175,7 +176,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), getFeatureAttributes(this.attributesBackend), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -203,7 +204,7 @@ LayergroupController.prototype.register = function(app) { forcedFormat ), getPreviewImageByCenter(this.previewBackend), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -229,7 +230,7 @@ LayergroupController.prototype.register = function(app) { forcedFormat ), getPreviewImageByBoundingBox(this.previewBackend), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -272,7 +273,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), getDataview(this.dataviewBackend), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -297,7 +298,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), getDataview(this.dataviewBackend), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -322,7 +323,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -347,7 +348,7 @@ LayergroupController.prototype.register = function(app) { this.layergroupAffectedTablesCache ), dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), @@ -610,14 +611,6 @@ function getPreviewImageByBoundingBox (previewBackend) { }; } -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=31536000'); - - next(); - }; -} - function incrementSuccessMetrics (statsClient) { return function incrementSuccessMetricsMiddleware (req, res, next) { const formatStat = parseFormat(req.params.format); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 670ed9d7..0caa61ce 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -11,6 +11,7 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const cacheControlHeader = require('../middleware/cache-control-header'); const cacheChannelHeader = require('../middleware/cache-channel-header'); const surrogateKeyHeader = require('../middleware/surrogate-key-header'); const lastModifiedHeader = require('../middleware/last-modified-header'); @@ -114,11 +115,11 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us this.getCreateMapMiddlewares(useTemplate), incrementMapViewCount(this.metadataBackend), augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader({ now: true }), setLastUpdatedTimeToLayergroup(), - setCacheControl(), setLayerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index c9d67c95..05dba8f8 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -6,6 +6,7 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const cacheControlHeader = require('../middleware/cache-control-header'); const cacheChannelHeader = require('../middleware/cache-channel-header'); const surrogateKeyHeader = require('../middleware/surrogate-key-header'); const lastModifiedHeader = require('../middleware/last-modified-header'); @@ -83,10 +84,10 @@ NamedMapsController.prototype.register = function(app) { tileBackend: this.tileBackend, label: 'NAMED_MAP_TILE' }), + cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), sendResponse(), vectorError() @@ -115,10 +116,10 @@ NamedMapsController.prototype.register = function(app) { getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), incrementMapViews({ metadataBackend: this.metadataBackend }), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + cacheControlHeader(), cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), sendResponse() ); @@ -444,24 +445,8 @@ function templateBounds(view) { return false; } -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware(req, res, next) { - const { affectedTables } = res.locals; - - res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - - if (!affectedTables || !!affectedTables.tables) { - // we increase cache control as we can invalidate it - res.set('Cache-Control', 'public,max-age=31536000'); - } - - next(); - }; - } - function setContentTypeHeader () { return function setContentTypeHeaderMiddleware(req, res, next) { - res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); next(); diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/middleware/cache-control-header.js new file mode 100644 index 00000000..25e2b04f --- /dev/null +++ b/lib/cartodb/middleware/cache-control-header.js @@ -0,0 +1,17 @@ +module.exports = function setCacheControlHeader ({ ttl = 31536000, revalidate = false } = {}) { + return function setCacheControlHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const directives = [ 'public', `max-age=${ttl}` ]; + + if (revalidate) { + directives.push('must-revalidate'); + } + + res.set('Cache-Control', directives.join(',')); + + next(); + }; +} diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index a1a93e40..148ec6db 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1272,6 +1272,8 @@ describe(suiteName, function() { it("cache control for layergroup default value", function(done) { global.environment.varnish.layergroupTtl = null; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=86400,must-revalidate'); @@ -1287,6 +1289,8 @@ describe(suiteName, function() { var layergroupTtl = 300; global.environment.varnish.layergroupTtl = layergroupTtl; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=' + layergroupTtl + ',must-revalidate'); From 52c8c9341ac296050e63a17642f02fc58ec5b287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Mar 2018 16:40:09 +0100 Subject: [PATCH 09/14] Remove function defined but never used --- lib/cartodb/controllers/map.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 0caa61ce..cdb1515f 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -442,17 +442,6 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) { }, lastUpdateTime); } -function setCacheControl () { - return function setCacheControlMiddleware (req, res, next) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - } - - next(); - }; -} - function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { const { user, mapConfig } = res.locals; From 4a2580c9ea20c0b07d6fef509d7849789441e4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Mar 2018 16:43:34 +0100 Subject: [PATCH 10/14] Missing semicolon --- lib/cartodb/middleware/cache-control-header.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/middleware/cache-control-header.js index 25e2b04f..aebb64de 100644 --- a/lib/cartodb/middleware/cache-control-header.js +++ b/lib/cartodb/middleware/cache-control-header.js @@ -14,4 +14,4 @@ module.exports = function setCacheControlHeader ({ ttl = 31536000, revalidate = next(); }; -} +}; From 672b19b106a2ca068196ded6684b3431d069e6eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Mar 2018 16:48:21 +0100 Subject: [PATCH 11/14] Magic number --- lib/cartodb/middleware/cache-control-header.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/middleware/cache-control-header.js index aebb64de..574ae3b5 100644 --- a/lib/cartodb/middleware/cache-control-header.js +++ b/lib/cartodb/middleware/cache-control-header.js @@ -1,4 +1,6 @@ -module.exports = function setCacheControlHeader ({ ttl = 31536000, revalidate = false } = {}) { +const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365; + +module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) { return function setCacheControlHeaderMiddleware (req, res, next) { if (req.method !== 'GET') { return next(); From b40ed13f477c23c69d05d77e03ab5fbe60f92d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Mar 2018 19:08:37 +0100 Subject: [PATCH 12/14] Do not use step to deal with asyn code --- .../provider/create-layergroup-provider.js | 84 +++++++++---------- .../mapconfig/provider/map-store-provider.js | 81 ++++++++---------- .../mapconfig/provider/named-map-provider.js | 82 ++++++++---------- 3 files changed, 108 insertions(+), 139 deletions(-) diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 0eef42f8..36298eb1 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -27,7 +27,13 @@ module.exports = CreateLayergroupMapConfigProvider; CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig && this.params && this.context) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -35,6 +41,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { function handleRenderLimits(err, renderLimits) { assert.ifError(err); context.limits = renderLimits; + self.context = context; return null; }, function finish(err) { @@ -52,59 +59,50 @@ CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.p CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey; CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) { - var self = this; + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } - const { dbname } = self.params; - const token = self.mapConfig.id(); + const { dbname } = this.params; + const token = mapConfig.id(); - if (self.affectedTablesCache.hasAffectedTables(dbname, token)) { - const affectedTables = self.affectedTablesCache.get(dbname, token); - return callback(null, affectedTables); - } + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } - step( - function getSql() { - const queries = []; + const queries = []; - self.mapConfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(table => { - queries.push(`SELECT * FROM ${table} LIMIT 0`); - }); - } - }); - - const sql = queries.length ? queries.join(';') : null; - - if (!sql) { - return callback(); + this.mapConfig.getLayers().forEach(function(layer) { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); } + }); - return sql; - }, - function getAffectedTables(err, sql) { - assert.ifError(err); + const sql = queries.length ? queries.join(';') : null; - step( - function getConnection() { - self.pgConnection.getConnection(self.user, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function finish(err, affectedTables) { + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { if (err) { return callback(err); } - self.affectedTablesCache.set(dbname, token, affectedTables); + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } - return callback(null, affectedTables); - } - ); + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(null, affectedTables); + }); + }); + }); }; diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index 2c94de92..ccf1472e 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -90,64 +90,51 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) { }; MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) { - var self = this; + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } - const { dbname, token } = self.params; + const { dbname } = this.params; + const token = mapConfig.id(); - if (self.affectedTablesCache.hasAffectedTables(dbname, token)) { - const affectedTables = self.affectedTablesCache.get(dbname, token); + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); - return callback(null, affectedTables); - } + return callback(null, affectedTables); + } - step( - function getMapConfig() { - self.getMapConfig(this); - }, - function getSql(err, mapConfig) { - assert.ifError(err); + const queries = []; - const queries = []; - - mapConfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(table => { - queries.push(`SELECT * FROM ${table} LIMIT 0`); - }); - } - }); - - const sql = queries.length ? queries.join(';') : null; - - if (!sql) { - return callback(); + mapConfig.getLayers().forEach(function(layer) { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); } + }); - return sql; - }, - function getAffectedTables(err, sql) { - assert.ifError(err); + const sql = queries.length ? queries.join(';') : null; - step( - function getConnection() { - self.pgConnection.getConnection(self.user, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function finish(err, affectedTables) { + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { if (err) { return callback(err); } - self.affectedTablesCache.set(dbname, token, affectedTables); + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } - return callback(err, affectedTables); - } - ); + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); }; diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 0fa003e2..50612065 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -263,66 +263,50 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() { }; NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) { - var self = this; + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } - let dbname = null; - let token = null; + const { dbname } = this.rendererParams; + const token = mapConfig.id(); - step( - function getMapConfig() { - self.getMapConfig(this); - }, - function getSql(err, mapConfig) { - assert.ifError(err); + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } - dbname = self.rendererParams; - token = mapConfig.id(); + const queries = []; - if (self.affectedTablesCache.hasAffectedTables(dbname, token)) { - const affectedTables = self.affectedTablesCache.get(dbname, token); - return callback(null, affectedTables); + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); } + }); - const queries = []; + const sql = queries.length ? queries.join(';') : null; - mapConfig.getLayers().forEach(layer => { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(table => { - queries.push(`SELECT * FROM ${table} LIMIT 0`); - }); - } - }); + if (!sql) { + return callback(); + } - const sql = queries.length ? queries.join(';') : null; - - if (!sql) { - return callback(); - } - - return sql; - }, - function getAffectedTables(err, sql) { - assert.ifError(err); - step( - function getConnection() { - self.pgConnection.getConnection(self.owner, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function finish(err, affectedTables) { + this.pgConnection.getConnection(this.owner, (err, connection) => { if (err) { return callback(err); } - self.affectedTablesCache.set(dbname, token, affectedTables); + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } - return callback(err, affectedTables); - } - ); + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); }; From e542d38ec7f44aa1fcbf686db90946923d907961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 22 Mar 2018 11:38:33 +0100 Subject: [PATCH 13/14] Reorder middleware --- lib/cartodb/controllers/named_maps.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 05dba8f8..26c5e4ad 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -84,11 +84,11 @@ NamedMapsController.prototype.register = function(app) { tileBackend: this.tileBackend, label: 'NAMED_MAP_TILE' }), + setContentTypeHeader(), cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), - setContentTypeHeader(), sendResponse(), vectorError() ); @@ -115,12 +115,12 @@ NamedMapsController.prototype.register = function(app) { }), getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), + setContentTypeHeader(), incrementMapViews({ metadataBackend: this.metadataBackend }), cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), - setContentTypeHeader(), sendResponse() ); }; @@ -388,6 +388,14 @@ function getImage({ previewBackend, label }) { }; } +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} + function incrementMapViewsError (ctx) { return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`; } @@ -444,11 +452,3 @@ function templateBounds(view) { } return false; } - -function setContentTypeHeader () { - return function setContentTypeHeaderMiddleware(req, res, next) { - res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); - - next(); - }; -} From a107ee67fac0a8ae9e30044cdc1957a1ddcff445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Mar 2018 10:32:22 +0200 Subject: [PATCH 14/14] Use arrow function --- .../models/mapconfig/provider/create-layergroup-provider.js | 2 +- lib/cartodb/models/mapconfig/provider/map-store-provider.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 36298eb1..b3b3f191 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -74,7 +74,7 @@ CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callba const queries = []; - this.mapConfig.getLayers().forEach(function(layer) { + this.mapConfig.getLayers().forEach(layer => { queries.push(layer.options.sql); if (layer.options.affected_tables) { layer.options.affected_tables.map(table => { diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index ccf1472e..f89a79ce 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -106,7 +106,7 @@ MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) { const queries = []; - mapConfig.getLayers().forEach(function(layer) { + mapConfig.getLayers().forEach(layer => { queries.push(layer.options.sql); if (layer.options.affected_tables) { layer.options.affected_tables.map(table => {