cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #914 from CartoDB/unify-middlewares

Browse Source 
Unify sendResponse middleware
This commit is contained in:
Daniel 2018-03-27 15:33:23 +02:00 committed by GitHub
commit 0aa2cffb5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 630 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
lib/cartodb/cache/named_map_provider_cache.js vendored
View File

@ -6,12 +6,20 @@ var queue = require('queue-async');
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
function NamedMapProviderCache(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache
) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.userLimitsApi = userLimitsApi;
this.mapConfigAdapter = mapConfigAdapter;
this.affectedTablesCache = affectedTablesCache;
this.providerCache = new LruCache({ max: 2000 });
}
@ -30,6 +38,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter,
this.affectedTablesCache,
user,
templateId,
config,

23
lib/cartodb/controllers/analyses.js
View File

@ -9,6 +9,8 @@ const authorize = require('../middleware/authorize');
const dbConnSetup = require('../middleware/db-conn-setup');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const cacheControlHeader = require('../middleware/cache-control-header');
const sendResponse = require('../middleware/send-response');
function AnalysesController(pgConnection, authApi, userLimitsApi) {
this.pgConnection = pgConnection;
@ -36,7 +38,7 @@ AnalysesController.prototype.register = function (app) {
getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),
getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }),
prepareResponse(),
setCacheControlHeader(),
cacheControlHeader({ ttl: 10, revalidate: true }),
sendResponse(),
unauthorizedError()
);
@ -111,25 +113,6 @@ function prepareResponse () {
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware (req, res, next) {
res.set('Cache-Control', 'public,max-age=10,must-revalidate');
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
res.status(200);
if (req.query && req.query.callback) {
res.jsonp(res.body);
} else {
res.json(res.body);
}
};
}
function unauthorizedError () {
return function unathorizedErrorMiddleware(err, req, res, next) {
if (err.message.match(/permission\sdenied/)) {

335
lib/cartodb/controllers/layergroup.js
View File

@ -9,10 +9,14 @@ const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const cacheControlHeader = require('../middleware/cache-control-header');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const DataviewBackend = require('../backends/dataview');
const AnalysisStatusBackend = require('../backends/analysis-status');
const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
const QueryTables = require('cartodb-query-tables');
const SUPPORTED_FORMATS = {
grid_json: true,
json_torque: true,
@ -43,7 +47,7 @@ function LayergroupController(
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
layergroupAffectedTablesCache,
analysisBackend,
authApi
) {
@ -54,7 +58,7 @@ function LayergroupController(
this.attributesBackend = attributesBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTables = layergroupAffectedTables;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.dataviewBackend = new DataviewBackend(analysisBackend);
this.analysisStatusBackend = new AnalysisStatusBackend();
@ -64,10 +68,10 @@ function LayergroupController(
module.exports = LayergroupController;
LayergroupController.prototype.register = function(app) {
const { base_url_mapconfig: mapconfigBasePath } = app;
const { base_url_mapconfig: mapConfigBasePath } = app;
app.get(
`${mapconfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
`${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
cors(),
cleanUpQueryParams(),
locals(),
@ -77,13 +81,17 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
@ -92,7 +100,7 @@ LayergroupController.prototype.register = function(app) {
);
app.get(
`${mapconfigBasePath}/:token/:z/:x/:y.:format`,
`${mapConfigBasePath}/:token/:z/:x/:y.:format`,
cors(),
cleanUpQueryParams(),
locals(),
@ -102,13 +110,17 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
@ -117,7 +129,7 @@ LayergroupController.prototype.register = function(app) {
);
app.get(
`${mapconfigBasePath}/:token/:layer/:z/:x/:y.(:format)`,
`${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`,
distinguishLayergroupFromStaticRoute(),
cors(),
cleanUpQueryParams(),
@ -128,13 +140,17 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'maplayer_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
@ -143,7 +159,7 @@ LayergroupController.prototype.register = function(app) {
);
app.get(
`${mapconfigBasePath}/:token/:layer/attributes/:fid`,
`${mapConfigBasePath}/:token/:layer/attributes/:fid`,
cors(),
cleanUpQueryParams(),
locals(),
@ -153,20 +169,24 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getFeatureAttributes(this.attributesBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
const forcedFormat = 'png';
app.get(
`${mapconfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
`${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
cors(),
cleanUpQueryParams(['layer']),
locals(),
@ -176,18 +196,23 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByCenter(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapconfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
`${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
cors(),
cleanUpQueryParams(['layer']),
locals(),
@ -197,13 +222,18 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByBoundingBox(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
@ -226,7 +256,7 @@ LayergroupController.prototype.register = function(app) {
];
app.get(
`${mapconfigBasePath}/:token/dataview/:dataviewName`,
`${mapConfigBasePath}/:token/dataview/:dataviewName`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@ -236,18 +266,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/:layer/widget/:dataviewName`,
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@ -257,18 +291,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/dataview/:dataviewName/search`,
`${mapConfigBasePath}/:token/dataview/:dataviewName/search`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@ -278,18 +316,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/:layer/widget/:dataviewName/search`,
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@ -299,18 +341,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/analysis/node/:nodeId`,
`${mapConfigBasePath}/:token/analysis/node/:nodeId`,
cors(),
cleanUpQueryParams(),
locals(),
@ -366,7 +412,13 @@ function getRequestParams(locals) {
return params;
}
function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) {
function createMapStoreMapConfigProvider (
mapStore,
userLimitsApi,
pgConnection,
affectedTablesCache,
forcedFormat = null
) {
return function createMapStoreMapConfigProviderMiddleware (req, res, next) {
const { user } = res.locals;
@ -377,7 +429,14 @@ function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat
params.layer = params.layer || 'all';
}
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params);
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(
mapStore,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
params
);
next();
};
@ -552,110 +611,6 @@ function getPreviewImageByBoundingBox (previewBackend) {
};
}
function setLastModifiedHeader () {
return function setLastModifiedHeaderMiddleware (req, res, next) {
let { cache_buster: cacheBuster } = res.locals;
cacheBuster = parseInt(cacheBuster, 10);
const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date();
res.set('Last-Modified', lastUpdated.toUTCString());
next();
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware (req, res, next) {
res.set('Cache-Control', 'public,max-age=31536000');
next();
};
}
function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) {
return function getAffectedTablesMiddleware (req, res, next) {
const { user, dbname, token } = res.locals;
if (layergroupAffectedTables.hasAffectedTables(dbname, token)) {
res.locals.affectedTables = layergroupAffectedTables.get(dbname, token);
return next();
}
mapStore.load(token, (err, mapconfig) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
const queries = [];
mapconfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
global.logger.warn('ERROR generating cache channel:' +
' this request doesn\'t need an X-Cache-Channel generated');
return next();
}
pgConnection.getConnection(user, (err, connection) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
req.profiler.done('getAffectedTablesFromQuery');
if (err) {
global.logger.warn('ERROR generating cache channel: ', err);
return next();
}
// feed affected tables cache so it can be reused from, for instance, map controller
layergroupAffectedTables.set(dbname, token, affectedTables);
res.locals.affectedTables = affectedTables;
next();
});
});
});
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
}
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
surrogateKeysCache.tag(res, affectedTables);
}
next();
};
}
function incrementSuccessMetrics (statsClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
const formatStat = parseFormat(req.params.format);
@ -667,24 +622,6 @@ function incrementSuccessMetrics (statsClient) {
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
res.status(res.statusCode || 200);
if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') {
if (req.query && req.query.callback) {
res.jsonp(res.body);
} else {
res.json(res.body);
}
} else {
res.send(res.body);
}
};
}
function incrementErrorMetrics (statsClient) {
return function incrementErrorMetricsMiddleware (err, req, res, next) {
const formatStat = parseFormat(req.params.format);

265
lib/cartodb/controllers/map.js
View File

@ -2,7 +2,6 @@ const _ = require('underscore');
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const QueryTables = require('cartodb-query-tables');
const ResourceLocator = require('../models/resource-locator');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
@ -12,7 +11,11 @@ const layergroupToken = require('../middleware/layergroup-token');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const cacheControlHeader = require('../middleware/cache-control-header');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
const LayergroupMetadata = require('../utils/layergroup-metadata');
@ -63,15 +66,15 @@ function MapController (
module.exports = MapController;
MapController.prototype.register = function(app) {
const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app;
const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app;
app.get(
`${mapconfigBasePath}`,
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.post(
`${mapconfigBasePath}`,
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
@ -87,7 +90,7 @@ MapController.prototype.register = function(app) {
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.options(app.base_url_mapconfig, cors('Content-Type'));
app.options(`${mapConfigBasePath}`, cors('Content-Type'));
};
MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) {
@ -112,11 +115,11 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us
this.getCreateMapMiddlewares(useTemplate),
incrementMapViewCount(this.metadataBackend),
augmentLayergroupData(),
getAffectedTables(this.pgConnection, this.layergroupAffectedTables),
setCacheChannel(),
setLastModified(),
cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader({ now: true }),
setLastUpdatedTimeToLayergroup(),
setCacheControl(),
setLayerStats(this.pgConnection, this.statsBackend),
setLayergroupIdHeader(this.templateMaps ,useTemplateHash),
setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata),
@ -124,7 +127,6 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us
setTurboCartoMetadataToLayergroup(this.layergroupMetadata),
setAggregationMetadataToLayergroup(this.layergroupMetadata),
setTilejsonMetadataToLayergroup(this.layergroupMetadata),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse(),
augmentError({ label, addContext })
];
@ -139,16 +141,27 @@ MapController.prototype.getCreateMapMiddlewares = function (useTemplate) {
this.pgConnection,
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter
this.mapConfigAdapter,
this.layergroupAffectedTables
),
instantiateLayergroup(this.mapBackend, this.userLimitsApi)
instantiateLayergroup(
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
}
return [
checkCreateLayergroup(),
prepareAdapterMapConfig(this.mapConfigAdapter),
createLayergroup (this.mapBackend, this.userLimitsApi)
createLayergroup (
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
};
@ -219,17 +232,25 @@ function checkCreateLayergroup () {
};
}
function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
function getTemplate (
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache
) {
return function getTemplateMiddleware (req, res, next) {
const templateParams = req.body;
const { user } = res.locals;
const mapconfigProvider = new NamedMapMapConfigProvider(
const mapConfigProvider = new NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache,
user,
req.params.template_id,
templateParams,
@ -237,15 +258,15 @@ function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi
res.locals
);
mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => {
mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => {
req.profiler.done('named.getMapConfig');
if (err) {
return next(err);
}
res.locals.mapconfig = mapconfig;
res.locals.mapConfig = mapConfig;
res.locals.rendererParams = rendererParams;
res.locals.mapconfigProvider = mapconfigProvider;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
@ -288,51 +309,63 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
};
}
function createLayergroup (mapBackend, userLimitsApi) {
function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function createLayergroupMiddleware (req, res, next) {
const requestMapConfig = req.body;
const { context, user } = res.locals;
const datasource = context.datasource || Datasource.EmptyDatasource();
const mapconfig = new MapConfig(requestMapConfig, datasource);
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals);
const mapConfig = new MapConfig(requestMapConfig, datasource);
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
res.locals
);
res.locals.mapconfig = mapconfig;
res.locals.mapConfig = mapConfig;
res.locals.analysesResults = context.analysesResults;
mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => {
mapBackend.createLayergroup(mapConfig, res.locals, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.locals.layergroup = layergroup;
res.body = layergroup;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
};
}
function instantiateLayergroup (mapBackend, userLimitsApi) {
function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function instantiateLayergroupMiddleware (req, res, next) {
const { user, mapconfig, rendererParams } = res.locals;
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams);
const { user, mapConfig, rendererParams } = res.locals;
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
rendererParams
);
mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => {
mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.locals.layergroup = layergroup;
res.body = layergroup;
const { mapconfigProvider } = res.locals;
const { mapConfigProvider } = res.locals;
res.locals.analysesResults = mapconfigProvider.analysesResults;
res.locals.template = mapconfigProvider.template;
res.locals.templateName = mapconfigProvider.getTemplateName();
res.locals.context = mapconfigProvider.context;
res.locals.analysesResults = mapConfigProvider.analysesResults;
res.locals.template = mapConfigProvider.template;
res.locals.context = mapConfigProvider.context;
next();
});
@ -341,10 +374,10 @@ function instantiateLayergroup (mapBackend, userLimitsApi) {
function incrementMapViewCount (metadataBackend) {
return function incrementMapViewCountMiddleware(req, res, next) {
const { mapconfig, user } = res.locals;
const { mapConfig, user } = res.locals;
// Error won't blow up, just be logged.
metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => {
metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => {
req.profiler.done('incMapviewCount');
if (err) {
@ -358,7 +391,7 @@ function incrementMapViewCount (metadataBackend) {
function augmentLayergroupData () {
return function augmentLayergroupDataMiddleware (req, res, next) {
const { layergroup } = res.locals;
const layergroup = res.body;
// include in layergroup response the variables in serverMedata
// those variables are useful to send to the client information
@ -369,80 +402,33 @@ function augmentLayergroupData () {
};
}
function getAffectedTables (pgConnection, layergroupAffectedTables) {
return function getAffectedTablesMiddleware (req, res, next) {
const { dbname, layergroup, user, mapconfig } = res.locals;
function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { mapConfigProvider, analysesResults } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
return next(err);
}
const sql = [];
mapconfig.getLayers().forEach(function(layer) {
sql.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
sql.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
if (!affectedTables) {
return next();
}
QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => {
req.profiler.done('getAffectedTablesFromQuery');
if (err) {
return next(err);
}
var lastUpdateTime = affectedTables.getLastUpdatedAt();
// feed affected tables cache so it can be reused from, for instance, layergroup controller
layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables);
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
res.locals.affectedTables = affectedTables;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
next();
});
next();
});
};
}
function setCacheChannel () {
return function setCacheChannelMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (req.method === 'GET') {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
}
next();
};
}
function setLastModified () {
return function setLastModifiedMiddleware (req, res, next) {
if (req.method === 'GET') {
res.set('Last-Modified', (new Date()).toUTCString());
}
next();
};
}
function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { affectedTables, layergroup, analysesResults } = res.locals;
var lastUpdateTime = affectedTables.getLastUpdatedAt();
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
next();
};
}
function getLastUpdatedTime(analysesResults, lastUpdateTime) {
if (!Array.isArray(analysesResults)) {
return lastUpdateTime;
@ -456,27 +442,17 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
}, lastUpdateTime);
}
function setCacheControl () {
return function setCacheControlMiddleware (req, res, next) {
if (req.method === 'GET') {
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
}
next();
};
}
function setLayerStats (pgConnection, statsBackend) {
return function setLayerStatsMiddleware(req, res, next) {
const { user, mapconfig, layergroup } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
if (err) {
return next(err);
}
statsBackend.getStats(mapconfig, connection, function(err, layersStats) {
statsBackend.getStats(mapConfig, connection, function(err, layersStats) {
if (err) {
return next(err);
}
@ -495,7 +471,8 @@ function setLayerStats (pgConnection, statsBackend) {
function setLayergroupIdHeader (templateMaps, useTemplateHash) {
return function setLayergroupIdHeaderMiddleware (req, res, next) {
const { layergroup, user, template } = res.locals;
const { user, template } = res.locals;
const layergroup = res.body;
if (useTemplateHash) {
var templateHash = templateMaps.fingerPrint(template).substring(0, 8);
@ -510,9 +487,10 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) {
function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) {
return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) {
const { layergroup, user, mapconfig } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj());
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj());
next();
};
@ -520,7 +498,8 @@ function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) {
function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) {
return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, user, analysesResults = [] } = res.locals;
const { user, analysesResults = [] } = res.locals;
const layergroup = res.body;
layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
@ -530,9 +509,10 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) {
function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, mapconfig, context } = res.locals;
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
@ -540,9 +520,10 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
function setAggregationMetadataToLayergroup (layergroupMetadata) {
return function setAggregationMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, mapconfig, context } = res.locals;
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
@ -550,54 +531,24 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) {
function setTilejsonMetadataToLayergroup (layergroupMetadata) {
return function augmentLayergroupTilejsonMiddleware (req, res, next) {
const { layergroup, user, mapconfig } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig);
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig);
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { affectedTables, user, templateName } = res.locals;
if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) {
surrogateKeysCache.tag(res, affectedTables);
}
if (templateName) {
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName));
}
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
const { layergroup } = res.locals;
res.status(200);
if (req.query && req.query.callback) {
res.jsonp(layergroup);
} else {
res.json(layergroup);
}
};
}
function augmentError (options) {
const { addContext = false, label = 'MAPS CONTROLLER' } = options;
return function augmentErrorMiddleware (err, req, res, next) {
req.profiler.done('error');
const { mapconfig } = res.locals;
const { mapConfig } = res.locals;
if (addContext) {
err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err;
err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err;
}
err.label = label;

171
lib/cartodb/controllers/named_maps.js
View File

@ -1,4 +1,3 @@
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
const locals = require('../middleware/locals');
@ -7,6 +6,11 @@ const layergroupToken = require('../middleware/layergroup-token');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const cacheControlHeader = require('../middleware/cache-control-header');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
@ -27,8 +31,8 @@ function getRequestParams(locals) {
const params = Object.assign({}, locals);
delete params.template;
delete params.affectedTablesAndLastUpdate;
delete params.namedMapProvider;
delete params.affectedTables;
delete params.mapConfigProvider;
delete params.allowedQueryParams;
return params;
@ -76,16 +80,15 @@ NamedMapsController.prototype.register = function(app) {
namedMapProviderCache: this.namedMapProviderCache,
label: 'NAMED_MAP_TILE'
}),
getAffectedTables(),
getTile({
tileBackend: this.tileBackend,
label: 'NAMED_MAP_TILE'
}),
setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
setCacheChannelHeader(),
setLastModifiedHeader(),
setCacheControlHeader(),
setContentTypeHeader(),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse(),
vectorError()
);
@ -105,7 +108,6 @@ NamedMapsController.prototype.register = function(app) {
namedMapProviderCache: this.namedMapProviderCache,
label: 'STATIC_VIZ_MAP', forcedFormat: 'png'
}),
getAffectedTables(),
getTemplate({ label: 'STATIC_VIZ_MAP' }),
prepareLayerFilterFromPreviewLayers({
namedMapProviderCache: this.namedMapProviderCache,
@ -113,12 +115,12 @@ NamedMapsController.prototype.register = function(app) {
}),
getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }),
getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }),
incrementMapViews({ metadataBackend: this.metadataBackend }),
setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
setCacheChannelHeader(),
setLastModifiedHeader(),
setCacheControlHeader(),
setContentTypeHeader(),
incrementMapViews({ metadataBackend: this.metadataBackend }),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
};
@ -142,25 +144,7 @@ function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = nul
return next(err);
}
res.locals.namedMapProvider = namedMapProvider;
next();
});
};
}
function getAffectedTables () {
return function getAffectedTables (req, res, next) {
const { namedMapProvider } = res.locals;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
req.profiler.done('affectedTables');
if (err) {
return next(err);
}
res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate;
res.locals.mapConfigProvider = namedMapProvider;
next();
});
@ -169,9 +153,9 @@ function getAffectedTables () {
function getTemplate ({ label }) {
return function getTemplateMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
const { mapConfigProvider } = res.locals;
namedMapProvider.getTemplate((err, template) => {
mapConfigProvider.getTemplate((err, template) => {
if (err) {
err.label = label;
return next(err);
@ -219,7 +203,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
return next(err);
}
res.locals.namedMapProvider = provider;
res.locals.mapConfigProvider = provider;
next();
});
@ -228,10 +212,11 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
function getTile ({ tileBackend, label }) {
return function getTileMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
const { mapConfigProvider, format } = res.locals;
tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => {
tileBackend.getTile(mapConfigProvider, req.params, (err, tile, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;
@ -251,7 +236,7 @@ function getTile ({ tileBackend, label }) {
function getStaticImageOptions ({ tablesExtentApi }) {
return function getStaticImageOptionsMiddleware(req, res, next) {
const { user, namedMapProvider, template } = res.locals;
const { user, mapConfigProvider, template } = res.locals;
const imageOpts = getImageOptions(res.locals, template);
@ -262,18 +247,18 @@ function getStaticImageOptions ({ tablesExtentApi }) {
res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
return next();
}
var affectedTables = affectedTablesAndLastUpdate.tables || [];
var tables = affectedTables.tables || [];
if (affectedTables.length === 0) {
if (tables.length === 0) {
return next();
}
tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => {
tablesExtentApi.getBounds(user, tables, (err, bounds) => {
if (err) {
return next();
}
@ -353,7 +338,7 @@ function getImageOptionsFromBoundingBox (bbox = '') {
function getImage({ previewBackend, label }) {
return function getImageMiddleware (req, res, next) {
const { imageOpts, namedMapProvider } = res.locals;
const { imageOpts, mapConfigProvider } = res.locals;
const { zoom, center, bounds } = imageOpts;
let { width, height } = req.params;
@ -364,7 +349,7 @@ function getImage({ previewBackend, label }) {
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
if (zoom !== undefined && center) {
return previewBackend.getImage(namedMapProvider, format, width, height, zoom, center,
return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center,
(err, image, headers, stats) => {
req.profiler.add(stats);
@ -383,8 +368,9 @@ function getImage({ previewBackend, label }) {
});
}
previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => {
previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;
@ -402,15 +388,23 @@ function getImage({ previewBackend, label }) {
};
}
function setContentTypeHeader () {
return function setContentTypeHeaderMiddleware(req, res, next) {
res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png');
next();
};
}
function incrementMapViewsError (ctx) {
return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
}
function incrementMapViews ({ metadataBackend }) {
return function incrementMapViewsMiddleware(req, res, next) {
const { user, namedMapProvider } = res.locals;
const { user, mapConfigProvider } = res.locals;
namedMapProvider.getMapConfig((err, mapConfig) => {
mapConfigProvider.getMapConfig((err, mapConfig) => {
if (err) {
global.logger.log(incrementMapViewsError({ user, err }));
return next();
@ -458,84 +452,3 @@ function templateBounds(view) {
}
return false;
}
function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals;
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName()));
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
if (affectedTablesAndLastUpdate.tables.length > 0) {
surrogateKeysCache.tag(res, affectedTablesAndLastUpdate);
}
}
next();
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel());
}
next();
};
}
function setLastModifiedHeader () {
return function setLastModifiedHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
var lastModifiedDate;
if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) {
lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
}
next();
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
}
next();
};
}
function setContentTypeHeader () {
return function setContentTypeHeaderMiddleware(req, res, next) {
res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png');
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
const { format } = res.locals;
req.profiler.done('render-' + format);
res.status(200);
res.send(res.body);
};
}

10
lib/cartodb/controllers/named_maps_admin.js
View File

@ -5,6 +5,7 @@ const locals = require('../middleware/locals');
const credentials = require('../middleware/credentials');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const sendResponse = require('../middleware/send-response');
/**
* @param {AuthApi} authApi
@ -224,12 +225,3 @@ function listTemplates ({ templateMaps }) {
});
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
res.status(res.statusCode || 200);
const method = req.query.callback ? 'jsonp' : 'json';
res[method](res.body);
};
}

24
lib/cartodb/middleware/cache-channel-header.js Normal file
View File

@ -0,0 +1,24 @@
module.exports = function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
if (req.method !== 'GET') {
return next();
}
const { mapConfigProvider } = res.locals;
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Cache Channel Header:', err);
return next();
}
if (!affectedTables) {
return next();
}
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
next();
});
};
};

19
lib/cartodb/middleware/cache-control-header.js Normal file
View File

@ -0,0 +1,19 @@
const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365;
module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) {
return function setCacheControlHeaderMiddleware (req, res, next) {
if (req.method !== 'GET') {
return next();
}
const directives = [ 'public', `max-age=${ttl}` ];
if (revalidate) {
directives.push('must-revalidate');
}
res.set('Cache-Control', directives.join(','));
next();
};
};

45
lib/cartodb/middleware/last-modified-header.js Normal file
View File

@ -0,0 +1,45 @@
module.exports = function setLastModifiedHeader ({ now = false } = {}) {
return function setLastModifiedHeaderMiddleware(req, res, next) {
if (req.method !== 'GET') {
return next();
}
const { mapConfigProvider, cache_buster } = res.locals;
if (cache_buster) {
const cacheBuster = parseInt(cache_buster, 10);
const lastModifiedDate = Number.isFinite(cacheBuster) ? new Date(cacheBuster) : new Date();
res.set('Last-Modified', lastModifiedDate.toUTCString());
return next();
}
// REVIEW: to keep 100% compatibility with maps controller
if (now) {
res.set('Last-Modified', new Date().toUTCString());
return next();
}
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Last Modified Header:', err);
return next();
}
if (!affectedTables) {
res.set('Last-Modified', new Date().toUTCString());
return next();
}
const lastUpdatedAt = affectedTables.getLastUpdatedAt();
const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? new Date(lastUpdatedAt) : new Date();
res.set('Last-Modified', lastModifiedDate.toUTCString());
next();
});
};
};

17
lib/cartodb/middleware/send-response.js Normal file
View File

@ -0,0 +1,17 @@
module.exports = function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
res.status(res.statusCode || 200);
if (Buffer.isBuffer(res.body)) {
return res.send(res.body);
}
if (req.query.callback) {
return res.jsonp(res.body);
}
res.json(res.body);
};
};

31
lib/cartodb/middleware/surrogate-key-header.js Normal file
View File

@ -0,0 +1,31 @@
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { user, mapConfigProvider } = res.locals;
if (mapConfigProvider instanceof NamedMapMapConfigProvider) {
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName()));
}
if (req.method !== 'GET') {
return next();
}
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Surrogate Key Header:', err);
return next();
}
if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) {
return next();
}
surrogateKeysCache.tag(res, affectedTables);
next();
});
};
};

62
lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js
View File

@ -2,6 +2,7 @@ var assert = require('assert');
var step = require('step');
var MapStoreMapConfigProvider = require('./map-store-provider');
const QueryTables = require('cartodb-query-tables');
/**
* @param {MapConfig} mapConfig
@ -11,10 +12,13 @@ var MapStoreMapConfigProvider = require('./map-store-provider');
* @constructor
* @type {CreateLayergroupMapConfigProvider}
*/
function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) {
function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
this.mapConfig = mapConfig;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.params = params;
this.cacheBuster = params.cache_buster || 0;
}
@ -23,7 +27,13 @@ module.exports = CreateLayergroupMapConfigProvider;
CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
var self = this;
if (this.mapConfig && this.params && this.context) {
return callback(null, this.mapConfig, this.params, this.context);
}
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
@ -31,6 +41,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);
context.limits = renderLimits;
self.context = context;
return null;
},
function finish(err) {
@ -46,3 +57,52 @@ CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigPr
CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter;
CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey;
CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
this.mapConfig.getLayers().forEach(layer => {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
this.pgConnection.getConnection(this.user, (err, connection) => {
if (err) {
return callback(err);
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
if (err) {
return callback(err);
}
this.affectedTablesCache.set(dbname, token, affectedTables);
callback(null, affectedTables);
});
});
});
};

69
lib/cartodb/models/mapconfig/provider/map-store-provider.js
View File

@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var dot = require('dot');
var step = require('step');
const QueryTables = require('cartodb-query-tables');
/**
* @param {MapStore} mapStore
@ -11,20 +12,30 @@ var step = require('step');
* @constructor
* @type {MapStoreMapConfigProvider}
*/
function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) {
function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
this.mapStore = mapStore;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.params = params;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.token = params.token;
this.cacheBuster = params.cache_buster || 0;
this.mapConfig = null;
this.params = params;
this.context = null;
}
module.exports = MapStoreMapConfigProvider;
MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
var self = this;
if (this.mapConfig !== null) {
return callback(null, this.mapConfig, this.params, this.context);
}
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
@ -39,6 +50,8 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
self.mapStore.load(self.token, this);
},
function finish(err, mapConfig) {
self.mapConfig = mapConfig;
self.context = context;
return callback(err, mapConfig, self.params, context);
}
);
@ -74,4 +87,54 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) {
scale_factor: 1
});
return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues);
};
};
MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
mapConfig.getLayers().forEach(layer => {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
this.pgConnection.getConnection(this.user, (err, connection) => {
if (err) {
return callback(err);
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
if (err) {
return callback(err);
}
this.affectedTablesCache.set(dbname, token, affectedTables);
callback(err, affectedTables);
});
});
});
};

99
lib/cartodb/models/mapconfig/provider/named-map-provider.js
View File

@ -11,8 +11,19 @@ var QueryTables = require('cartodb-query-tables');
* @constructor
* @type {NamedMapMapConfigProvider}
*/
function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter,
owner, templateId, config, authToken, params) {
function NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache,
owner,
templateId,
config,
authToken,
params
) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
@ -30,7 +41,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend,
// use template after call to mapConfig
this.template = null;
this.affectedTablesAndLastUpdate = null;
this.affectedTablesCache = affectedTablesCache;
// providing
this.err = null;
@ -189,7 +200,7 @@ NamedMapMapConfigProvider.prototype.getCacheBuster = function() {
NamedMapMapConfigProvider.prototype.reset = function() {
this.template = null;
this.affectedTablesAndLastUpdate = null;
this.affectedTables = null;
this.err = null;
this.mapConfig = null;
@ -251,39 +262,51 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() {
return this.templateName;
};
NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) {
var self = this;
if (this.affectedTablesAndLastUpdate !== null) {
return callback(null, this.affectedTablesAndLastUpdate);
}
step(
function getMapConfig() {
self.getMapConfig(this);
},
function getSql(err, mapConfig) {
assert.ifError(err);
return mapConfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
},
function getAffectedTables(err, sql) {
assert.ifError(err);
step(
function getConnection() {
self.pgConnection.getConnection(self.owner, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;
return callback(err, result);
NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
);
const { dbname } = this.rendererParams;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
mapConfig.getLayers().forEach(layer => {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
this.pgConnection.getConnection(this.owner, (err, connection) => {
if (err) {
return callback(err);
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
if (err) {
return callback(err);
}
this.affectedTablesCache.set(dbname, token, affectedTables);
callback(err, affectedTables);
});
});
});
};

3
lib/cartodb/server.js
View File

@ -200,7 +200,8 @@ module.exports = function(serverOptions) {
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter
mapConfigAdapter,
layergroupAffectedTablesCache
);
['update', 'delete'].forEach(function(eventType) {

4
test/acceptance/multilayer.js
View File

@ -1272,6 +1272,8 @@ describe(suiteName, function() {
it("cache control for layergroup default value", function(done) {
global.environment.varnish.layergroupTtl = null;
var server = new CartodbWindshaft(serverOptions);
assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation,
function(res) {
assert.equal(res.headers['cache-control'], 'public,max-age=86400,must-revalidate');
@ -1287,6 +1289,8 @@ describe(suiteName, function() {
var layergroupTtl = 300;
global.environment.varnish.layergroupTtl = layergroupTtl;
var server = new CartodbWindshaft(serverOptions);
assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation,
function(res) {
assert.equal(res.headers['cache-control'], 'public,max-age=' + layergroupTtl + ',must-revalidate');