cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into rateLimits

Browse Source
This commit is contained in:
Simon Martín 2018-03-13 18:09:44 +01:00
commit 0e43c54214
32 changed files with 1241 additions and 926 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
NEWS.md
View File

@ -2,9 +2,12 @@
## 5.4.0
Released yyyy-mm-dd
- Upgrades Windshaft to 4.5.3
- Upgrades Windshaft to 4.5.6 ([Mapnik top metrics](https://github.com/CartoDB/Windshaft/pull/597), [AttributesBackend allows multiple features if all the attributes are the same](https://github.com/CartoDB/Windshaft/pull/602))
- Implemented middleware to authorize users via new Api Key system
- Keep the old authorization system as fallback
- Aggregation widget: Remove NULL categories in 'count' aggregations too
- Update request to 2.85.0
- Update camshaft to 0.61.3 (Fixes duplicated cartodb_id in AOI)
## 5.3.1
Released 2018-02-13

8
lib/cartodb/api/auth_api.js
View File

@ -62,7 +62,7 @@ function isValidApiKey(apikey) {
//
AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) {
const apikeyToken = res.locals.api_key;
const apikeyUsername = res.locals.apikeyUsername;
const basicAuthUsername = res.locals.basicAuthUsername;
if ( ! apikeyToken ) {
return callback(null, false); // no api key, no authorization...
@ -91,7 +91,7 @@ AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) {
return callback(error);
}
if (!usernameMatches(apikeyUsername, res.locals.user)) {
if (!usernameMatches(basicAuthUsername, res.locals.user)) {
const error = new Error('Forbidden');
error.type = 'auth';
error.subtype = 'api-key-username-mismatch';
@ -149,8 +149,8 @@ function isNameNotFoundError (err) {
return err.message && -1 !== err.message.indexOf('name not found');
}
function usernameMatches (apikeyUsername, requestUsername) {
return !(apikeyUsername && (apikeyUsername !== requestUsername));
function usernameMatches (basicAuthUsername, requestUsername) {
return !(basicAuthUsername && (basicAuthUsername !== requestUsername));
}
/**

2
lib/cartodb/controllers/analyses.js
View File

@ -12,7 +12,7 @@ AnalysesController.prototype.register = function (app) {
app.get(
`${app.base_url_mapconfig}/analyses/catalog`,
cors(),
userMiddleware,
userMiddleware(),
this.prepareContext,
this.createPGClient(),
this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),

793
lib/cartodb/controllers/layergroup.js
View File

@ -1,22 +1,24 @@
var assert = require('assert');
var step = require('step');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var allowQueryParams = require('../middleware/allow-query-params');
var vectorError = require('../middleware/vector-error');
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
var DataviewBackend = require('../backends/dataview');
var AnalysisStatusBackend = require('../backends/analysis-status');
var MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
var QueryTables = require('cartodb-query-tables');
const DataviewBackend = require('../backends/dataview');
const AnalysisStatusBackend = require('../backends/analysis-status');
const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
const QueryTables = require('cartodb-query-tables');
const SUPPORTED_FORMATS = {
grid_json: true,
json_torque: true,
torque_json: true,
png: true,
png32: true,
mvt: true
};
/**
* @param {AuthApi} authApi
* @param {prepareContext} prepareContext
* @param {PgConnection} pgConnection
* @param {MapStore} mapStore
* @param {TileBackend} tileBackend
@ -58,70 +60,125 @@ function LayergroupController(
module.exports = LayergroupController;
LayergroupController.prototype.register = function(app) {
const { base_url_mapconfig: basePath } = app;
app.get(
app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format',
`${basePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
this.tile.bind(this),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError()
);
app.get(
app.base_url_mapconfig + '/:token/:z/:x/:y.:format',
`${basePath}/:token/:z/:x/:y.:format`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
this.tile.bind(this),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError()
);
app.get(
app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)',
`${basePath}/:token/:layer/:z/:x/:y.(:format)`,
distinguishLayergroupFromStaticRoute(),
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
validateLayerRouteMiddleware,
this.prepareContext,
this.layer.bind(this),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'maplayer_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError()
);
app.get(
app.base_url_mapconfig + '/:token/:layer/attributes/:fid',
`${basePath}/:token/:layer/attributes/:fid`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
this.prepareContext,
this.attributes.bind(this)
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getFeatureAttributes(this.attributesBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
const forcedFormat = 'png';
app.get(
app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
`${basePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
allowQueryParams(['layer']),
this.prepareContext,
this.center.bind(this)
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
getPreviewImageByCenter(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
`${basePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
allowQueryParams(['layer']),
this.prepareContext,
this.bbox.bind(this)
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
getPreviewImageByBoundingBox(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
var allowedDataviewQueryParams = [
const allowedDataviewQueryParams = [
'filters', // json
'own_filter', // 0, 1
'no_filters', // 0, 1
@ -137,400 +194,470 @@ LayergroupController.prototype.register = function(app) {
];
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName',
`${basePath}/:token/dataview/:dataviewName`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
this.dataview.bind(this)
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
`${basePath}/:token/:layer/widget/:dataviewName`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
this.dataview.bind(this)
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
`${basePath}/:token/dataview/:dataviewName/search`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
this.dataviewSearch.bind(this)
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
`${basePath}/:token/:layer/widget/:dataviewName/search`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
this.dataviewSearch.bind(this)
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
app.base_url_mapconfig + '/:token/analysis/node/:nodeId',
`${basePath}/:token/analysis/node/:nodeId`,
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
this.prepareContext,
this.analysisNodeStatus.bind(this)
analysisNodeStatus(this.analysisStatusBackend),
sendResponse()
);
};
LayergroupController.prototype.analysisNodeStatus = function(req, res, next) {
var self = this;
function distinguishLayergroupFromStaticRoute () {
return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) {
if (req.params.token === 'static') {
return next('route');
}
step(
function retrieveNodeStatus() {
self.analysisStatusBackend.getNodeStatus(res.locals, this);
},
function finish(err, nodeStatus, stats) {
req.profiler.add(stats || {});
next();
};
}
function analysisNodeStatus (analysisStatusBackend) {
return function analysisNodeStatusMiddleware(req, res, next) {
analysisStatusBackend.getNodeStatus(res.locals, (err, nodeStatus, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET NODE STATUS';
next(err);
} else {
self.sendResponse(req, res, nodeStatus, 200, {
return next(err);
}
res.set({
'Cache-Control': 'public,max-age=5',
'Last-Modified': new Date().toUTCString()
});
}
}
);
};
LayergroupController.prototype.dataview = function(req, res, next) {
var self = this;
res.body = nodeStatus;
step(
function retrieveDataview() {
var mapConfigProvider = new MapStoreMapConfigProvider(
self.mapStore, res.locals.user, self.userLimitsApi, res.locals
);
self.dataviewBackend.getDataview(
mapConfigProvider,
res.locals.user,
res.locals,
this
);
},
function finish(err, dataview, stats) {
req.profiler.add(stats || {});
next();
});
};
}
function getRequestParams(locals) {
const params = Object.assign({}, locals);
delete params.mapConfigProvider;
delete params.allowedQueryParams;
return params;
}
function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) {
return function createMapStoreMapConfigProviderMiddleware (req, res, next) {
const { user } = res.locals;
const params = getRequestParams(res.locals);
if (forcedFormat) {
params.format = forcedFormat;
params.layer = params.layer || 'all';
}
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params);
next();
};
}
function getDataview (dataviewBackend) {
return function getDataviewMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW';
next(err);
} else {
self.sendResponse(req, res, dataview, 200);
return next(err);
}
}
);
};
LayergroupController.prototype.dataviewSearch = function(req, res, next) {
var self = this;
res.body = dataview;
step(
function searchDataview() {
var mapConfigProvider = new MapStoreMapConfigProvider(
self.mapStore, res.locals.user, self.userLimitsApi, res.locals
);
self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this);
},
function finish(err, searchResult, stats) {
req.profiler.add(stats || {});
next();
});
};
}
function dataviewSearch (dataviewBackend) {
return function dataviewSearchMiddleware (req, res, next) {
const { user, dataviewName, mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW SEARCH';
next(err);
} else {
self.sendResponse(req, res, searchResult, 200);
return next(err);
}
}
);
};
res.body = searchResult;
LayergroupController.prototype.attributes = function(req, res, next) {
var self = this;
next();
});
};
}
function getFeatureAttributes (attributesBackend) {
return function getFeatureAttributesMiddleware (req, res, next) {
req.profiler.start('windshaft.maplayer_attribute');
step(
function retrieveFeatureAttributes() {
var mapConfigProvider = new MapStoreMapConfigProvider(
self.mapStore, res.locals.user, self.userLimitsApi, res.locals
);
self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this);
},
function finish(err, tile, stats) {
req.profiler.add(stats || {});
const { mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET ATTRIBUTES';
next(err);
} else {
self.sendResponse(req, res, tile, 200);
return next(err);
}
}
);
};
res.body = tile;
// Gets a tile for a given token and set of tile ZXY coords. (OSM style)
LayergroupController.prototype.tile = function(req, res, next) {
req.profiler.start('windshaft.map_tile');
this.tileOrLayer(req, res, next);
};
// Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
LayergroupController.prototype.layer = function(req, res, next) {
req.profiler.start('windshaft.maplayer_tile');
this.tileOrLayer(req, res, next);
};
LayergroupController.prototype.tileOrLayer = function (req, res, next) {
var self = this;
step(
function mapController$getTileOrGrid() {
self.tileBackend.getTile(
new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
res.locals, this
);
},
function mapController$finalize(err, tile, headers, stats) {
req.profiler.add(stats);
self.finalizeGetTileOrGrid(err, req, res, tile, headers, next);
}
);
};
function getStatusCode(tile, format){
return tile.length===0 && format==='mvt'? 204:200;
next();
});
};
}
// This function is meant for being called as the very last
// step by all endpoints serving tiles or grids
LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) {
var supportedFormats = {
grid_json: true,
json_torque: true,
torque_json: true,
png: true,
png32: true,
mvt: true
};
function getStatusCode(tile, format){
return tile.length === 0 && format === 'mvt'? 204 : 200;
}
var formatStat = 'invalid';
if (req.params.format) {
var format = req.params.format.replace('.', '_');
if (supportedFormats[format]) {
formatStat = format;
}
}
function parseFormat (format = '') {
const prettyFormat = format.replace('.', '_');
return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid';
}
function getTile (tileBackend, profileLabel = 'tile') {
return function getTileMiddleware (req, res, next) {
req.profiler.start(`windshaft.${profileLabel}`);
const { mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
req.profiler.add(stats);
if (err) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var errMsg = err.message ? ( '' + err.message ) : ( '' + err );
// Rewrite mapnik parsing errors to start with layer number
var matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
if (matches) {
errMsg = 'style'+matches[2]+': ' + matches[1];
}
err.message = errMsg;
err.label = 'TILE RENDER';
next(err);
global.statsClient.increment('windshaft.tiles.error');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
} else {
this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers);
global.statsClient.increment('windshaft.tiles.success');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
}
};
LayergroupController.prototype.bbox = function(req, res, next) {
this.staticMap(req, res, +req.params.width, +req.params.height, {
west: +req.params.west,
north: +req.params.north,
east: +req.params.east,
south: +req.params.south
}, null, next);
};
LayergroupController.prototype.center = function(req, res, next) {
this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
lng: +req.params.lng,
lat: +req.params.lat
}, next);
};
LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) {
var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
// We force always the tile to be generated using PNG because
// is the only format we support by now
res.locals.format = 'png';
res.locals.layer = res.locals.layer || 'all';
var self = this;
step(
function getImage() {
if (center) {
self.previewBackend.getImage(
new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
format, width, height, zoom, center, this);
} else {
self.previewBackend.getImage(
new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
format, width, height, zoom /* bounds */, this);
}
},
function handleImage(err, image, headers, stats) {
req.profiler.done('render-' + format);
req.profiler.add(stats || {});
if (err) {
err.label = 'STATIC_MAP';
next(err);
} else {
res.set('Content-Type', headers['Content-Type'] || 'image/' + format);
self.sendResponse(req, res, image, 200);
}
}
);
};
LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
var self = this;
req.profiler.done('res');
res.set('Cache-Control', 'public,max-age=31536000');
// Set Last-Modified header
var lastUpdated;
if (res.locals.cache_buster) {
// Assuming cache_buster is a timestamp
lastUpdated = new Date(parseInt(res.locals.cache_buster));
} else {
lastUpdated = new Date();
}
res.set('Last-Modified', lastUpdated.toUTCString());
var dbName = res.locals.dbname;
step(
function getAffectedTables() {
self.getAffectedTables(res.locals.user, dbName, res.locals.token, this);
},
function sendResponse(err, affectedTables) {
req.profiler.done('affectedTables');
if (err) {
global.logger.warn('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
self.surrogateKeysCache.tag(res, affectedTables);
return next(err);
}
if (headers) {
res.set(headers);
}
res.status(status);
const formatStat = parseFormat(req.params.format);
if (!Buffer.isBuffer(body) && typeof body === 'object') {
if (req.query && req.query.callback) {
res.jsonp(body);
} else {
res.json(body);
}
} else {
res.send(body);
}
}
);
};
res.statusCode = getStatusCode(tile, formatStat);
res.body = tile;
LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
next();
});
};
}
if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) {
return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId));
function getPreviewImageByCenter (previewBackend) {
return function getPreviewImageByCenterMiddleware (req, res, next) {
const width = +req.params.width;
const height = +req.params.height;
const zoom = +req.params.z;
const center = {
lng: +req.params.lng,
lat: +req.params.lat
};
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
const { mapConfigProvider: provider } = res.locals;
previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {
err.label = 'STATIC_MAP';
return next(err);
}
var self = this;
step(
function extractSQL() {
step(
function loadFromStore() {
self.mapStore.load(layergroupId, this);
},
function getSQL(err, mapConfig) {
assert.ifError(err);
if (headers) {
res.set(headers);
}
var queries = [];
mapConfig.getLayers().forEach(function(layer) {
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.body = image;
next();
});
};
}
function getPreviewImageByBoundingBox (previewBackend) {
return function getPreviewImageByBoundingBoxMiddleware (req, res, next) {
const width = +req.params.width;
const height = +req.params.height;
const bounds = {
west: +req.params.west,
north: +req.params.north,
east: +req.params.east,
south: +req.params.south
};
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
const { mapConfigProvider: provider } = res.locals;
previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {
err.label = 'STATIC_MAP';
return next(err);
}
if (headers) {
res.set(headers);
}
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.body = image;
next();
});
};
}
function setLastModifiedHeader () {
return function setLastModifiedHeaderMiddleware (req, res, next) {
let { cache_buster: cacheBuster } = res.locals;
cacheBuster = parseInt(cacheBuster, 10);
const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date();
res.set('Last-Modified', lastUpdated.toUTCString());
next();
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware (req, res, next) {
res.set('Cache-Control', 'public,max-age=31536000');
next();
};
}
function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) {
return function getAffectedTablesMiddleware (req, res, next) {
const { user, dbname, token } = res.locals;
if (layergroupAffectedTables.hasAffectedTables(dbname, token)) {
res.locals.affectedTables = layergroupAffectedTables.get(dbname, token);
return next();
}
mapStore.load(token, (err, mapconfig) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
const queries = [];
mapconfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
queries.push('SELECT * FROM ' + table + ' LIMIT 0');
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
const sql = queries.length ? queries.join(';') : null;
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
if (!sql) {
global.logger.warn('ERROR generating cache channel:' +
' this request doesn\'t need an X-Cache-Channel generated');
return next();
}
step(
function getConnection() {
self.pgConnection.getConnection(user, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
pgConnection.getConnection(user, (err, connection) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function buildCacheChannel(err, tables) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables);
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
req.profiler.done('getAffectedTablesFromQuery');
if (err) {
global.logger.warn('ERROR generating cache channel: ', err);
return next();
}
return tables;
},
callback
);
};
// feed affected tables cache so it can be reused from, for instance, map controller
layergroupAffectedTables.set(dbname, token, affectedTables);
res.locals.affectedTables = affectedTables;
function validateLayerRouteMiddleware(req, res, next) {
if (req.params.token === 'static') {
return next('route');
next();
});
});
});
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
}
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
surrogateKeysCache.tag(res, affectedTables);
}
next();
};
}
function incrementSuccessMetrics (statsClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.success');
statsClient.increment(`windshaft.tiles.${formatStat}.success`);
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
res.status(res.statusCode || 200);
if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') {
if (req.query && req.query.callback) {
res.jsonp(res.body);
} else {
res.json(res.body);
}
} else {
res.send(res.body);
}
};
}
function incrementErrorMetrics (statsClient) {
return function incrementErrorMetricsMiddleware (err, req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.error');
statsClient.increment(`windshaft.tiles.${formatStat}.error`);
next(err);
};
}
function tileError () {
return function tileErrorMiddleware (err, req, res, next) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
let errMsg = err.message ? ( '' + err.message ) : ( '' + err );
// Rewrite mapnik parsing errors to start with layer number
const matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
if (matches) {
errMsg = `style${matches[2]}: ${matches[1]}`;
}
err.message = errMsg;
err.label = 'TILE RENDER';
next(err);
};
}

427
lib/cartodb/controllers/map.js
View File

@ -1,23 +1,19 @@
var _ = require('underscore');
var windshaft = require('windshaft');
var QueryTables = require('cartodb-query-tables');
var ResourceLocator = require('../models/resource-locator');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
const _ = require('underscore');
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const QueryTables = require('cartodb-query-tables');
const ResourceLocator = require('../models/resource-locator');
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
const LayergroupMetadata = require('../utils/layergroup-metadata');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
/**
* @param {AuthApi} authApi
* @param {PgConnection} pgConnection
@ -43,7 +39,8 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m
this.layergroupAffectedTables = layergroupAffectedTables;
this.mapConfigAdapter = mapConfigAdapter;
this.resourceLocator = new ResourceLocator(global.environment);
const resourceLocator = new ResourceLocator(global.environment);
this.layergroupMetadata = new LayergroupMetadata(resourceLocator);
this.statsBackend = statsBackend;
this.prepareContext = prepareContext;
@ -86,36 +83,56 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us
return [
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, endpointGroup),
allowQueryParams(['aggregation']),
this.prepareContext,
this.initProfiler(isTemplateInstantiation),
this.checkJsonContentType(),
useTemplate ? this.checkInstantiteLayergroup() : this.checkCreateLayergroup(),
useTemplate ? this.getTemplate() : this.prepareAdapterMapConfig(),
useTemplate ? this.instantiateLayergroup() : this.createLayergroup(),
this.incrementMapViewCount(),
this.augmentLayergroupData(),
this.getAffectedTables(),
this.setCacheChannel(),
this.setLastModified(),
this.setLastUpdatedTimeToLayergroup(),
this.setCacheControl(),
this.setLayerStats(),
this.setLayergroupIdHeader(useTemplateHash),
this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(),
this.setAnalysesMetadataToLayergroup(includeQuery),
this.setTurboCartoMetadataToLayergroup(),
this.setAggregationMetadataToLayergroup(),
this.setTilejsonMetadataToLayergroup(),
this.setSurrogateKeyHeader(),
this.sendResponse(),
this.augmentError({ label, addContext })
initProfiler(isTemplateInstantiation),
checkJsonContentType(),
this.getCreateMapMiddlewares(useTemplate),
incrementMapViewCount(this.metadataBackend),
augmentLayergroupData(),
getAffectedTables(this.pgConnection, this.layergroupAffectedTables),
setCacheChannel(),
setLastModified(),
setLastUpdatedTimeToLayergroup(),
setCacheControl(),
setLayerStats(this.pgConnection, this.statsBackend),
setLayergroupIdHeader(this.templateMaps ,useTemplateHash),
setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata),
setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery),
setTurboCartoMetadataToLayergroup(this.layergroupMetadata),
setAggregationMetadataToLayergroup(this.layergroupMetadata),
setTilejsonMetadataToLayergroup(this.layergroupMetadata),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse(),
augmentError({ label, addContext })
];
};
MapController.prototype.initProfiler = function (isTemplateInstantiation) {
MapController.prototype.getCreateMapMiddlewares = function (useTemplate) {
if (useTemplate) {
return [
checkInstantiteLayergroup(),
getTemplate(
this.templateMaps,
this.pgConnection,
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter
),
instantiateLayergroup(this.mapBackend, this.userLimitsApi)
];
}
return [
checkCreateLayergroup(),
prepareAdapterMapConfig(this.mapConfigAdapter),
createLayergroup (this.mapBackend, this.userLimitsApi)
];
};
function initProfiler (isTemplateInstantiation) {
const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
return function initProfilerMiddleware (req, res, next) {
@ -123,9 +140,9 @@ MapController.prototype.initProfiler = function (isTemplateInstantiation) {
req.profiler.done(`${operation}.initProfilerMiddleware`);
next();
};
};
}
MapController.prototype.checkJsonContentType = function () {
function checkJsonContentType () {
return function checkJsonContentTypeMiddleware(req, res, next) {
if (req.method === 'POST' && !req.is('application/json')) {
return next(new Error('POST data must be of type application/json'));
@ -135,9 +152,9 @@ MapController.prototype.checkJsonContentType = function () {
next();
};
};
}
MapController.prototype.checkInstantiteLayergroup = function () {
function checkInstantiteLayergroup () {
return function checkInstantiteLayergroupMiddleware(req, res, next) {
if (req.method === 'GET') {
const { callback, config } = req.query;
@ -159,9 +176,9 @@ MapController.prototype.checkInstantiteLayergroup = function () {
return next();
};
};
}
MapController.prototype.checkCreateLayergroup = function () {
function checkCreateLayergroup () {
return function checkCreateLayergroupMiddleware (req, res, next) {
if (req.method === 'GET') {
const { config } = res.locals;
@ -180,19 +197,19 @@ MapController.prototype.checkCreateLayergroup = function () {
req.profiler.done('checkCreateLayergroup');
return next();
};
};
}
MapController.prototype.getTemplate = function () {
function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
return function getTemplateMiddleware (req, res, next) {
const templateParams = req.body;
const { user } = res.locals;
const mapconfigProvider = new NamedMapMapConfigProvider(
this.templateMaps,
this.pgConnection,
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter,
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
user,
req.params.template_id,
templateParams,
@ -212,10 +229,10 @@ MapController.prototype.getTemplate = function () {
next();
});
}.bind(this);
};
};
}
MapController.prototype.prepareAdapterMapConfig = function () {
function prepareAdapterMapConfig (mapConfigAdapter) {
return function prepareAdapterMapConfigMiddleware(req, res, next) {
const requestMapConfig = req.body;
const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals;
@ -237,7 +254,7 @@ MapController.prototype.prepareAdapterMapConfig = function () {
}
};
this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => {
mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => {
req.profiler.done('anonymous.getMapConfig');
if (err) {
return next(err);
@ -248,22 +265,22 @@ MapController.prototype.prepareAdapterMapConfig = function () {
next();
});
}.bind(this);
};
};
}
MapController.prototype.createLayergroup = function () {
function createLayergroup (mapBackend, userLimitsApi) {
return function createLayergroupMiddleware (req, res, next) {
const requestMapConfig = req.body;
const { context, user } = res.locals;
const datasource = context.datasource || Datasource.EmptyDatasource();
const mapconfig = new MapConfig(requestMapConfig, datasource);
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals);
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals);
res.locals.mapconfig = mapconfig;
res.locals.analysesResults = context.analysesResults;
this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => {
mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
@ -273,16 +290,16 @@ MapController.prototype.createLayergroup = function () {
next();
});
}.bind(this);
};
};
}
MapController.prototype.instantiateLayergroup = function () {
function instantiateLayergroup (mapBackend, userLimitsApi) {
return function instantiateLayergroupMiddleware (req, res, next) {
const { user, mapconfig, rendererParams } = res.locals;
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams);
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams);
this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => {
mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
@ -299,15 +316,15 @@ MapController.prototype.instantiateLayergroup = function () {
next();
});
}.bind(this);
};
};
}
MapController.prototype.incrementMapViewCount = function () {
function incrementMapViewCount (metadataBackend) {
return function incrementMapViewCountMiddleware(req, res, next) {
const { mapconfig, user } = res.locals;
// Error won't blow up, just be logged.
this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => {
metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => {
req.profiler.done('incMapviewCount');
if (err) {
@ -316,10 +333,10 @@ MapController.prototype.incrementMapViewCount = function () {
next();
});
}.bind(this);
};
};
}
MapController.prototype.augmentLayergroupData = function () {
function augmentLayergroupData () {
return function augmentLayergroupDataMiddleware (req, res, next) {
const { layergroup } = res.locals;
@ -330,91 +347,13 @@ MapController.prototype.augmentLayergroupData = function () {
next();
};
};
function getTemplateUrl(url) {
return url.https || url.http;
}
function getTilejson(tiles, grids) {
const tilejson = {
tilejson: '2.2.0',
tiles: tiles.https || tiles.http
};
if (grids) {
tilejson.grids = grids.https || grids.http;
}
return tilejson;
}
MapController.prototype.setTilejsonMetadataToLayergroup = function () {
return function augmentLayergroupTilejsonMiddleware (req, res, next) {
const { layergroup, user, mapconfig } = res.locals;
const isVectorOnlyMapConfig = mapconfig.isVectorOnlyMapConfig();
let hasMapnikLayers = false;
layergroup.metadata.layers.forEach((layerMetadata, index) => {
const layerId = mapconfig.getLayerId(index);
const rasterResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.png`;
if (mapconfig.layerType(index) === 'mapnik') {
hasMapnikLayers = true;
const vectorResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.mvt`;
const layerTilejson = {
vector: getTilejson(this.resourceLocator.getTileUrls(user, vectorResource))
};
if (!isVectorOnlyMapConfig) {
let grids = null;
const layer = mapconfig.getLayer(index);
if (layer.options.interactivity) {
const gridResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.grid.json`;
grids = this.resourceLocator.getTileUrls(user, gridResource);
}
layerTilejson.raster = getTilejson(
this.resourceLocator.getTileUrls(user, rasterResource),
grids
);
}
layerMetadata.tilejson = layerTilejson;
} else {
layerMetadata.tilejson = {
raster: getTilejson(this.resourceLocator.getTileUrls(user, rasterResource))
};
}
});
const tilejson = {};
const url = {};
if (hasMapnikLayers) {
const vectorResource = `${layergroup.layergroupid}/{z}/{x}/{y}.mvt`;
tilejson.vector = getTilejson(
this.resourceLocator.getTileUrls(user, vectorResource)
);
url.vector = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, vectorResource));
if (!isVectorOnlyMapConfig) {
const rasterResource = `${layergroup.layergroupid}/{z}/{x}/{y}.png`;
tilejson.raster = getTilejson(
this.resourceLocator.getTileUrls(user, rasterResource)
);
url.raster = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, rasterResource));
}
}
layergroup.metadata.tilejson = tilejson;
layergroup.metadata.url = url;
next();
}.bind(this);
};
MapController.prototype.getAffectedTables = function () {
function getAffectedTables (pgConnection, layergroupAffectedTables) {
return function getAffectedTablesMiddleware (req, res, next) {
const { dbname, layergroup, user, mapconfig } = res.locals;
this.pgConnection.getConnection(user, (err, connection) => {
pgConnection.getConnection(user, (err, connection) => {
if (err) {
return next(err);
}
@ -436,17 +375,17 @@ MapController.prototype.getAffectedTables = function () {
}
// feed affected tables cache so it can be reused from, for instance, layergroup controller
this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables);
layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables);
res.locals.affectedTables = affectedTables;
next();
});
});
}.bind(this);
};
};
}
MapController.prototype.setCacheChannel = function () {
function setCacheChannel () {
return function setCacheChannelMiddleware (req, res, next) {
const { affectedTables } = res.locals;
@ -456,9 +395,9 @@ MapController.prototype.setCacheChannel = function () {
next();
};
};
}
MapController.prototype.setLastModified = function () {
function setLastModified () {
return function setLastModifiedMiddleware (req, res, next) {
if (req.method === 'GET') {
res.set('Last-Modified', (new Date()).toUTCString());
@ -466,9 +405,9 @@ MapController.prototype.setLastModified = function () {
next();
};
};
}
MapController.prototype.setLastUpdatedTimeToLayergroup = function () {
function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { affectedTables, layergroup, analysesResults } = res.locals;
@ -482,7 +421,7 @@ MapController.prototype.setLastUpdatedTimeToLayergroup = function () {
next();
};
};
}
function getLastUpdatedTime(analysesResults, lastUpdateTime) {
if (!Array.isArray(analysesResults)) {
@ -497,7 +436,7 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
}, lastUpdateTime);
}
MapController.prototype.setCacheControl = function () {
function setCacheControl () {
return function setCacheControlMiddleware (req, res, next) {
if (req.method === 'GET') {
var ttl = global.environment.varnish.layergroupTtl || 86400;
@ -506,18 +445,18 @@ MapController.prototype.setCacheControl = function () {
next();
};
};
}
MapController.prototype.setLayerStats = function () {
function setLayerStats (pgConnection, statsBackend) {
return function setLayerStatsMiddleware(req, res, next) {
const { user, mapconfig, layergroup } = res.locals;
this.pgConnection.getConnection(user, (err, connection) => {
pgConnection.getConnection(user, (err, connection) => {
if (err) {
return next(err);
}
this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) {
statsBackend.getStats(mapconfig, connection, function(err, layersStats) {
if (err) {
return next(err);
}
@ -531,171 +470,91 @@ MapController.prototype.setLayerStats = function () {
next();
});
});
}.bind(this);
};
};
}
MapController.prototype.setLayergroupIdHeader = function (useTemplateHash) {
function setLayergroupIdHeader (templateMaps, useTemplateHash) {
return function setLayergroupIdHeaderMiddleware (req, res, next) {
const { layergroup, user, template } = res.locals;
if (useTemplateHash) {
var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8);
var templateHash = templateMaps.fingerPrint(template).substring(0, 8);
layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`;
}
res.set('X-Layergroup-Id', layergroup.layergroupid);
next();
}.bind(this);
};
};
}
MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function () {
function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) {
return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) {
const { layergroup, user, mapconfig } = res.locals;
this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj());
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj());
next();
}.bind(this);
};
// TODO this should take into account several URL patterns
MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) {
this.addDataviewsUrls(username, layergroup, mapConfig);
this.addWidgetsUrl(username, layergroup, mapConfig);
};
MapController.prototype.addDataviewsUrls = function(username, layergroup, mapConfig) {
layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
var dataviews = mapConfig.dataviews || {};
Object.keys(dataviews).forEach(function(dataviewName) {
var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
layergroup.metadata.dataviews[dataviewName] = {
url: this.resourceLocator.getUrls(username, resource)
};
}.bind(this));
};
}
MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
var mapConfigLayer = mapConfig.layers[layerIndex];
if (mapConfigLayer.options && mapConfigLayer.options.widgets) {
layer.widgets = layer.widgets || {};
Object.keys(mapConfigLayer.options.widgets).forEach(function(widgetName) {
var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
layer.widgets[widgetName] = {
type: mapConfigLayer.options.widgets[widgetName].type,
url: this.resourceLocator.getUrls(username, resource)
};
}.bind(this));
}
return layer;
}.bind(this));
}
};
MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) {
function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) {
return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, user, analysesResults = [] } = res.locals;
this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
next();
}.bind(this);
};
MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) {
includeQuery = includeQuery || false;
analysesResults = analysesResults || [];
layergroup.metadata.analyses = [];
analysesResults.forEach(function(analysis) {
var nodes = analysis.getNodes();
layergroup.metadata.analyses.push({
nodes: nodes.reduce(function(nodesIdMap, node) {
if (node.params.id) {
var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
var nodeRepr = {
status: node.getStatus(),
url: this.resourceLocator.getUrls(username, nodeResource)
};
if (includeQuery) {
nodeRepr.query = node.getQuery();
}
if (node.getStatus() === 'failed') {
nodeRepr.error_message = node.getErrorMessage();
}
nodesIdMap[node.params.id] = nodeRepr;
}
}
return nodesIdMap;
}.bind(this), {})
});
}.bind(this));
};
MapController.prototype.setTurboCartoMetadataToLayergroup = function () {
function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, mapconfig, context } = res.locals;
addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context);
next();
};
};
function addTurboCartoContextMetadata(layergroup, mapConfig, context) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
}
return layer;
});
}
}
// TODO: see how evolve this function, it's a good candidate to be refactored
MapController.prototype.setAggregationMetadataToLayergroup = function () {
function setAggregationMetadataToLayergroup (layergroupMetadata) {
return function setAggregationMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, mapconfig, context } = res.locals;
addAggregationContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context);
next();
};
};
function addAggregationContextMetadata(layergroup, mapConfig, context) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
if (context.aggregation && Array.isArray(context.aggregation.layers)) {
layer.meta.aggregation = context.aggregation.layers[layerIndex];
}
return layer;
});
}
}
MapController.prototype.setSurrogateKeyHeader = function () {
function setTilejsonMetadataToLayergroup (layergroupMetadata) {
return function augmentLayergroupTilejsonMiddleware (req, res, next) {
const { layergroup, user, mapconfig } = res.locals;
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig);
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { affectedTables, user, templateName } = res.locals;
if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) {
this.surrogateKeysCache.tag(res, affectedTables);
surrogateKeysCache.tag(res, affectedTables);
}
if (templateName) {
this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName));
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName));
}
next();
}.bind(this);
};
};
}
MapController.prototype.sendResponse = function () {
function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
const { layergroup } = res.locals;
@ -708,9 +567,9 @@ MapController.prototype.sendResponse = function () {
res.json(layergroup);
}
};
};
}
MapController.prototype.augmentError = function (options) {
function augmentError (options) {
const { addContext = false, label = 'MAPS CONTROLLER' } = options;
return function augmentErrorMiddleware (err, req, res, next) {
@ -725,7 +584,7 @@ MapController.prototype.augmentError = function (options) {
next(err);
};
};
}
function populateError(err, mapConfig) {
var error = new Error(err.message);

41
lib/cartodb/controllers/named_maps.js
View File

@ -52,15 +52,19 @@ function NamedMapsController(
module.exports = NamedMapsController;
NamedMapsController.prototype.register = function(app) {
const tileOptions = {
label: 'NAMED_MAP_TILE'
};
app.get(
app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)',
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES),
this.prepareContext,
this.getNamedMapProvider('NAMED_MAP_TILE'),
this.getNamedMapProvider(tileOptions),
this.getAffectedTables(),
this.getTile('NAMED_MAP_TILE'),
this.getTile(tileOptions),
this.setSurrogateKey(),
this.setCacheChannelHeader(),
this.setLastModifiedHeader(),
@ -70,19 +74,24 @@ NamedMapsController.prototype.register = function(app) {
vectorError()
);
const staticOptions = {
forcedFormat: 'png',
label: 'STATIC_VIZ_MAP'
};
app.get(
app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
cors(),
userMiddleware,
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED),
allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
this.prepareContext,
this.getNamedMapProvider('STATIC_VIZ_MAP'),
this.getNamedMapProvider(staticOptions),
this.getAffectedTables(),
this.getTemplate('STATIC_VIZ_MAP'),
this.prepareLayerFilterFromPreviewLayers('STATIC_VIZ_MAP'),
this.getTemplate(staticOptions),
this.prepareLayerFilterFromPreviewLayers(staticOptions),
this.getStaticImageOptions(),
this.getImage('STATIC_VIZ_MAP'),
this.getImage(staticOptions),
this.incrementMapViews(),
this.setSurrogateKey(),
this.setCacheChannelHeader(),
@ -93,16 +102,16 @@ NamedMapsController.prototype.register = function(app) {
);
};
NamedMapsController.prototype.getNamedMapProvider = function (label) {
NamedMapsController.prototype.getNamedMapProvider = function ({ label, forcedFormat = null }) {
return function getNamedMapProviderMiddleware (req, res, next) {
const { user } = res.locals;
const { config, auth_token } = req.query;
const { template_id } = req.params;
// We force always the tile to be generated using PNG because
// is the only format we support by now
res.locals.format = 'png';
if (forcedFormat) {
res.locals.format = forcedFormat;
res.locals.layer = res.locals.layer || 'all';
}
const params = getRequestParams(res.locals);
@ -137,7 +146,7 @@ NamedMapsController.prototype.getAffectedTables = function () {
}.bind(this);
};
NamedMapsController.prototype.getTemplate = function (label) {
NamedMapsController.prototype.getTemplate = function ({ label }) {
return function getTemplateMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
@ -154,7 +163,7 @@ NamedMapsController.prototype.getTemplate = function (label) {
};
};
NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (label) {
NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function ({ label }) {
return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) {
const { user, template } = res.locals;
const { template_id } = req.params;
@ -196,7 +205,7 @@ NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (la
}.bind(this);
};
NamedMapsController.prototype.getTile = function (label) {
NamedMapsController.prototype.getTile = function ({ label }) {
return function getTileMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
@ -319,7 +328,7 @@ function getImageOptionsFromBoundingBox (bbox = '') {
}
}
NamedMapsController.prototype.getImage = function (label) {
NamedMapsController.prototype.getImage = function ({ label }) {
return function getImageMiddleware (req, res, next) {
const { imageOpts, namedMapProvider } = res.locals;
const { zoom, center, bounds } = imageOpts;

32
lib/cartodb/controllers/named_maps_admin.js
View File

@ -4,12 +4,7 @@ const userMiddleware = require('../middleware/user');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const localsMiddleware = require('../middleware/context/locals');
const apikeyCredentialsMiddleware = require('../middleware/context/apikey-credentials');
const apikeyMiddleware = [
localsMiddleware,
apikeyCredentialsMiddleware(),
];
const credentialsMiddleware = require('../middleware/context/credentials');
/**
* @param {AuthApi} authApi
@ -31,9 +26,10 @@ NamedMapsAdminController.prototype.register = function (app) {
app.post(
`${base_url_templated}/`,
cors(),
userMiddleware,
userMiddleware(),
localsMiddleware(),
credentialsMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE),
apikeyMiddleware,
this.checkContentType('POST', 'POST TEMPLATE'),
this.authorizedByAPIKey('create', 'POST TEMPLATE'),
this.create()
@ -42,9 +38,10 @@ NamedMapsAdminController.prototype.register = function (app) {
app.put(
`${base_url_templated}/:template_id`,
cors(),
userMiddleware,
userMiddleware(),
localsMiddleware(),
credentialsMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE),
apikeyMiddleware,
this.checkContentType('PUT', 'PUT TEMPLATE'),
this.authorizedByAPIKey('update', 'PUT TEMPLATE'),
this.update()
@ -53,9 +50,10 @@ NamedMapsAdminController.prototype.register = function (app) {
app.get(
`${base_url_templated}/:template_id`,
cors(),
userMiddleware,
userMiddleware(),
localsMiddleware(),
credentialsMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET),
apikeyMiddleware,
this.authorizedByAPIKey('get', 'GET TEMPLATE'),
this.retrieve()
);
@ -63,9 +61,10 @@ NamedMapsAdminController.prototype.register = function (app) {
app.delete(
`${base_url_templated}/:template_id`,
cors(),
userMiddleware,
userMiddleware(),
localsMiddleware(),
credentialsMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE),
apikeyMiddleware,
this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'),
this.destroy()
);
@ -73,9 +72,10 @@ NamedMapsAdminController.prototype.register = function (app) {
app.get(
`${base_url_templated}/`,
cors(),
userMiddleware,
userMiddleware(),
localsMiddleware(),
credentialsMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST),
apikeyMiddleware,
this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'),
this.list()
);

5
lib/cartodb/middleware/allow-query-params.js
View File

@ -1,8 +1,9 @@
module.exports = function allowQueryParams(params) {
module.exports = function allowQueryParams (params) {
if (!Array.isArray(params)) {
throw new Error('allowQueryParams must receive an Array of params');
}
return function allowQueryParamsMiddleware(req, res, next) {
return function allowQueryParamsMiddleware (req, res, next) {
res.locals.allowedQueryParams = params;
next();
};

7
lib/cartodb/middleware/context/authorize.js
View File

@ -1,9 +1,8 @@
module.exports = function authorizeMiddleware (authApi) {
return function (req, res, next) {
req.profiler.done('req2params.setup');
module.exports = function authorize (authApi) {
return function authorizeMiddleware (req, res, next) {
authApi.authorize(req, res, (err, authorized) => {
req.profiler.done('authorize');
if (err) {
return next(err);
}

15
lib/cartodb/middleware/context/apikey-credentials.js → lib/cartodb/middleware/context/credentials.js
View File

@ -1,18 +1,17 @@
'use strict';
const basicAuth = require('basic-auth');
module.exports = function apikeyToken () {
return function apikeyTokenMiddleware(req, res, next) {
module.exports = function credentials () {
return function credentialsMiddleware(req, res, next) {
const apikeyCredentials = getApikeyCredentialsFromRequest(req);
res.locals.api_key = apikeyCredentials.token;
res.locals.apikeyUsername = apikeyCredentials.username;
res.locals.basicAuthUsername = apikeyCredentials.username;
res.set('vary', 'Authorization'); //Honor Authorization header when caching.
return next();
};
};
//--------------------------------------------------------------------------------
const basicAuth = require('basic-auth');
function getApikeyCredentialsFromRequest(req) {
let apikeyCredentials = {
token: null,

15
lib/cartodb/middleware/context/db-conn-setup.js
View File

@ -1,14 +1,17 @@
const _ = require('underscore');
module.exports = function dbConnSetupMiddleware(pgConnection) {
return function dbConnSetup(req, res, next) {
const user = res.locals.user;
module.exports = function dbConnSetup (pgConnection) {
return function dbConnSetupMiddleware (req, res, next) {
const { user } = res.locals;
pgConnection.setDBConn(user, res.locals, (err) => {
req.profiler.done('dbConnSetup');
if (err) {
if (err.message && -1 !== err.message.indexOf('name not found')) {
err.http_status = 404;
}
req.profiler.done('req2params');
return next(err);
}
@ -21,9 +24,7 @@ module.exports = function dbConnSetupMiddleware(pgConnection) {
res.set('X-Served-By-DB-Host', res.locals.dbhost);
req.profiler.done('req2params');
next(null);
next();
});
};
};

8
lib/cartodb/middleware/context/index.js
View File

@ -1,16 +1,16 @@
const locals = require('./locals');
const cleanUpQueryParams = require('./clean-up-query-params');
const layergroupToken = require('./layergroup-token');
const apikeyCredentials = require('./apikey-credentials');
const credentials = require('./credentials');
const authorize = require('./authorize');
const dbConnSetup = require('./db-conn-setup');
module.exports = function prepareContextMiddleware(authApi, pgConnection) {
return [
locals,
locals(),
cleanUpQueryParams(),
layergroupToken,
apikeyCredentials(),
layergroupToken(),
credentials(),
authorize(authApi),
dbConnSetup(pgConnection)
];

29
lib/cartodb/middleware/context/layergroup-token.js
View File

@ -1,32 +1,33 @@
var LayergroupToken = require('../../models/layergroup-token');
const LayergroupToken = require('../../models/layergroup-token');
const authErrorMessageTemplate = function (signer, user) {
return `Cannot use map signature of user "${signer}" on db of user "${user}"`;
};
module.exports = function layergroupTokenMiddleware(req, res, next) {
module.exports = function layergroupToken () {
return function layergroupTokenMiddleware (req, res, next) {
if (!res.locals.token) {
return next();
}
var user = res.locals.user;
const user = res.locals.user;
const layergroupToken = LayergroupToken.parse(res.locals.token);
var layergroupToken = LayergroupToken.parse(res.locals.token);
res.locals.token = layergroupToken.token;
res.locals.cache_buster = layergroupToken.cacheBuster;
if (layergroupToken.signer) {
res.locals.signer = layergroupToken.signer;
if (!res.locals.signer) {
res.locals.signer = user;
} else if (res.locals.signer !== user) {
var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`);
err.type = 'auth';
err.http_status = 403;
if (req.query && req.query.callback) {
err.http_status = 200;
}
req.profiler.done('req2params');
if (res.locals.signer !== user) {
const err = new Error(authErrorMessageTemplate(res.locals.signer, user));
err.type = 'auth';
err.http_status = (req.query && req.query.callback) ? 200: 403;
return next(err);
}
}
return next();
};
};

5
lib/cartodb/middleware/context/locals.js
View File

@ -1,6 +1,7 @@
module.exports = function localsMiddleware(req, res, next) {
// save req.params in res.locals
module.exports = function locals () {
return function localsMiddleware (req, res, next) {
res.locals = Object.assign(req.params || {}, res.locals);
next();
};
};

9
lib/cartodb/middleware/cors.js
View File

@ -1,11 +1,14 @@
module.exports = function cors(extraHeaders) {
return function(req, res, next) {
var baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token";
module.exports = function cors (extraHeaders) {
return function corsMiddleware (req, res, next) {
let baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token";
if(extraHeaders) {
baseHeaders += ", " + extraHeaders;
}
res.set("Access-Control-Allow-Origin", "*");
res.set("Access-Control-Allow-Headers", baseHeaders);
next();
};
};

11
lib/cartodb/middleware/lzma.js
View File

@ -1,10 +1,9 @@
'use strict';
const LZMA = require('lzma').LZMA;
const lzmaWorker = new LZMA();
module.exports = function lzma () {
const lzmaWorker = new LZMA();
module.exports = function lzmaMiddleware(req, res, next) {
return function lzmaMiddleware (req, res, next) {
if (!req.query.hasOwnProperty('lzma')) {
return next();
}
@ -22,9 +21,13 @@ module.exports = function lzmaMiddleware(req, res, next) {
try {
delete req.query.lzma;
Object.assign(req.query, JSON.parse(result));
req.profiler.done('lzma');
next();
} catch (err) {
next(new Error('Error parsing lzma as JSON: ' + err));
}
});
};
};

4
lib/cartodb/middleware/stats.js
View File

@ -2,10 +2,10 @@ const Profiler = require('../stats/profiler_proxy');
const debug = require('debug')('windshaft:cartodb:stats');
const onHeaders = require('on-headers');
module.exports = function statsMiddleware(options) {
module.exports = function stats (options) {
const { enabled = true, statsClient } = options;
return function stats(req, res, next) {
return function statsMiddleware (req, res, next) {
req.profiler = new Profiler({
statsd_client: statsClient,
profile: enabled

9
lib/cartodb/middleware/user.js
View File

@ -1,8 +1,11 @@
var CdbRequest = require('../models/cdb_request');
var cdbRequest = new CdbRequest();
const CdbRequest = require('../models/cdb_request');
module.exports = function userMiddleware(req, res, next) {
module.exports = function user () {
const cdbRequest = new CdbRequest();
return function userMiddleware(req, res, next) {
res.locals.user = cdbRequest.userByReq(req);
next();
};
};

1
lib/cartodb/middleware/vector-error.js
View File

@ -1,5 +1,4 @@
const fs = require('fs');
const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt');
module.exports = function vectorError() {

2
lib/cartodb/models/dataview/aggregation.js
View File

@ -42,7 +42,7 @@ const rankedCategoriesQueryTpl = ctx => `
${ctx.aggregationFn} AS value,
row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
FROM (${filteredQueryTpl(ctx)}) filtered_source
${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
WHERE ${ctx.aggregation === "count" ? `${ctx.column}` : `${ctx.aggregationColumn}`} IS NOT NULL
GROUP BY ${ctx.column}
ORDER BY 2 DESC
)

2
lib/cartodb/server.js
View File

@ -379,7 +379,7 @@ function bootstrap(opts) {
statsClient: global.statsClient
}));
app.use(lzmaMiddleware);
app.use(lzmaMiddleware());
// temporary measure until we upgrade to newer version expressjs so we can check err.status
app.use(function(err, req, res, next) {

168
lib/cartodb/utils/layergroup-metadata.js Normal file
View File

@ -0,0 +1,168 @@
module.exports = class LayergroupMetadata {
constructor (resourceLocator) {
this.resourceLocator = resourceLocator;
}
// TODO this should take into account several URL patterns
addDataviewsAndWidgetsUrls (username, layergroup, mapConfig) {
this._addDataviewsUrls(username, layergroup, mapConfig);
this._addWidgetsUrl(username, layergroup, mapConfig);
}
_addDataviewsUrls (username, layergroup, mapConfig) {
layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
var dataviews = mapConfig.dataviews || {};
Object.keys(dataviews).forEach((dataviewName) => {
var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
layergroup.metadata.dataviews[dataviewName] = {
url: this.resourceLocator.getUrls(username, resource)
};
});
}
_addWidgetsUrl (username, layergroup, mapConfig) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map((layer, layerIndex) => {
var mapConfigLayer = mapConfig.layers[layerIndex];
if (mapConfigLayer.options && mapConfigLayer.options.widgets) {
layer.widgets = layer.widgets || {};
Object.keys(mapConfigLayer.options.widgets).forEach((widgetName) => {
var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
layer.widgets[widgetName] = {
type: mapConfigLayer.options.widgets[widgetName].type,
url: this.resourceLocator.getUrls(username, resource)
};
});
}
return layer;
});
}
}
addAnalysesMetadata (username, layergroup, analysesResults, includeQuery) {
includeQuery = includeQuery || false;
analysesResults = analysesResults || [];
layergroup.metadata.analyses = [];
analysesResults.forEach((analysis) => {
var nodes = analysis.getNodes();
layergroup.metadata.analyses.push({
nodes: nodes.reduce((nodesIdMap, node) => {
if (node.params.id) {
var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
var nodeRepr = {
status: node.getStatus(),
url: this.resourceLocator.getUrls(username, nodeResource)
};
if (includeQuery) {
nodeRepr.query = node.getQuery();
}
if (node.getStatus() === 'failed') {
nodeRepr.error_message = node.getErrorMessage();
}
nodesIdMap[node.params.id] = nodeRepr;
}
return nodesIdMap;
}, {})
});
});
}
addAggregationContextMetadata (layergroup, mapConfig, context) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
if (context.aggregation && Array.isArray(context.aggregation.layers)) {
layer.meta.aggregation = context.aggregation.layers[layerIndex];
}
return layer;
});
}
}
addTileJsonMetadata (layergroup, user, mapconfig) {
const isVectorOnlyMapConfig = mapconfig.isVectorOnlyMapConfig();
let hasMapnikLayers = false;
layergroup.metadata.layers.forEach((layerMetadata, index) => {
const layerId = mapconfig.getLayerId(index);
const rasterResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.png`;
if (mapconfig.layerType(index) === 'mapnik') {
hasMapnikLayers = true;
const vectorResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.mvt`;
const layerTilejson = {
vector: this._getTilejson(this.resourceLocator.getTileUrls(user, vectorResource))
};
if (!isVectorOnlyMapConfig) {
let grids = null;
const layer = mapconfig.getLayer(index);
if (layer.options.interactivity) {
const gridResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.grid.json`;
grids = this.resourceLocator.getTileUrls(user, gridResource);
}
layerTilejson.raster = this._getTilejson(
this.resourceLocator.getTileUrls(user, rasterResource),
grids
);
}
layerMetadata.tilejson = layerTilejson;
} else {
layerMetadata.tilejson = {
raster: this._getTilejson(this.resourceLocator.getTileUrls(user, rasterResource))
};
}
});
const tilejson = {};
const url = {};
if (hasMapnikLayers) {
const vectorResource = `${layergroup.layergroupid}/{z}/{x}/{y}.mvt`;
tilejson.vector = this._getTilejson(
this.resourceLocator.getTileUrls(user, vectorResource)
);
url.vector = this._getTemplateUrl(this.resourceLocator.getTemplateUrls(user, vectorResource));
if (!isVectorOnlyMapConfig) {
const rasterResource = `${layergroup.layergroupid}/{z}/{x}/{y}.png`;
tilejson.raster = this._getTilejson(
this.resourceLocator.getTileUrls(user, rasterResource)
);
url.raster = this._getTemplateUrl(this.resourceLocator.getTemplateUrls(user, rasterResource));
}
}
layergroup.metadata.tilejson = tilejson;
layergroup.metadata.url = url;
}
_getTilejson(tiles, grids) {
const tilejson = {
tilejson: '2.2.0',
tiles: tiles.https || tiles.http
};
if (grids) {
tilejson.grids = grids.https || grids.http;
}
return tilejson;
}
_getTemplateUrl(url) {
return url.https || url.http;
}
addTurboCartoContextMetadata(layergroup, mapConfig, context) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
}
return layer;
});
}
}
};

6
package.json
View File

@ -26,7 +26,7 @@
"dependencies": {
"basic-auth": "^2.0.0",
"body-parser": "^1.18.2",
"camshaft": "0.61.2",
"camshaft": "0.61.3",
"cartodb-psql": "0.10.2",
"cartodb-query-tables": "0.3.0",
"cartodb-redis": "cartodb/node-cartodb-redis#rateLimits",
@ -42,13 +42,13 @@
"on-headers": "^1.0.1",
"queue-async": "~1.0.7",
"redis-mpool": "0.5.0",
"request": "^2.83.0",
"request": "2.85.0",
"semver": "~5.3.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"turbo-carto": "0.20.2",
"underscore": "~1.6.0",
"windshaft": "4.5.3",
"windshaft": "4.5.6",
"yargs": "~5.0.0"
},
"devDependencies": {

1
test/acceptance/cache/cache_headers.js vendored
View File

@ -155,6 +155,7 @@ describe('get requests with cache headers', function() {
assert.ok(res.headers['x-cache-channel']);
assert.ok(res.headers['surrogate-key']);
assert.equal(res.headers.vary, 'Authorization');
if (expectedCacheHeaders) {
validateXChannelHeaders(res.headers, expectedCacheHeaders);
assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys);

89
test/acceptance/dataviews/aggregation.js
View File

@ -70,12 +70,8 @@ describe('aggregations happy cases', function() {
].join(' UNION ALL ');
operations.forEach(function (operation) {
var not = operation === 'count' ? ' not ' : ' ';
var description = 'should' +
not +
'handle NULL values in category and aggregation columns using "' +
operation +
'" as aggregation operation';
var description = 'should handle NULL values in category and aggregation columns using "' +
operation + '" as aggregation operation';
it(description, function (done) {
this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val'));
@ -96,12 +92,7 @@ describe('aggregations happy cases', function() {
}
});
if (operation === 'count') {
assert.ok(hasNullCategory, 'aggregation has not a category NULL');
} else {
assert.ok(!hasNullCategory, 'aggregation has category NULL');
}
done();
});
});
@ -425,3 +416,79 @@ describe('aggregation dataview tuned by categories query param', function () {
});
});
});
describe('Count aggregation', function () {
const mapConfig = {
version: '1.5.0',
layers: [
{
type: "cartodb",
options: {
source: {
"id": "a0"
},
cartocss: "#points { marker-width: 10; marker-fill: red; }",
cartocss_version: "2.3.0"
}
}
],
dataviews: {
categories: {
source: {
id: 'a0'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'count'
}
}
},
analyses: [
{
id: "a0",
type: "source",
params: {
query: `
SELECT
null::geometry the_geom_webmercator,
CASE
WHEN x % 4 = 0 THEN 1
WHEN x % 4 = 1 THEN 2
WHEN x % 4 = 2 THEN 3
ELSE null
END AS val,
CASE
WHEN x % 4 = 0 THEN 'category_1'
WHEN x % 4 = 1 THEN 'category_2'
WHEN x % 4 = 2 THEN 'category_3'
ELSE null
END AS cat
FROM generate_series(1, 1000) x
`
}
}
]
};
it(`should handle null values correctly when aggregationColumn isn't provided`, function (done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => {
assert.ifError(err);
assert.equal(dataview.categories.length, 3);
this.testClient.drain(done);
});
});
it(`should handle null values correctly when aggregationColumn is provided`, function (done) {
mapConfig.dataviews.categories.options.aggregationColumn = 'val';
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => {
assert.ifError(err);
assert.equal(dataview.categories.length, 3);
this.testClient.drain(done);
});
});
});

42
test/acceptance/mvt.js
View File

@ -34,6 +34,48 @@ return function () {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
});
describe('named map tile', function () {
it('should get default named vector tile', function (done) {
const apikeyToken = 1234;
const templateName = `mvt-template-${usePostGIS ? 'postgis' : 'mapnik'}`;
const template = {
version: '0.0.1',
name: templateName,
placeholders: {
buffersize: {
type: 'number',
default: 0
}
},
layergroup: {
version: '1.7.0',
layers: [{
type: 'cartodb',
options: {
sql: 'select * from populated_places_simple_reduced limit 10',
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0',
}
}]
}
};
const testClient = new TestClient(template, apikeyToken);
testClient.getNamedTile(templateName, 0, 0, 0, 'mvt', {}, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
assert.equal(tileJSON[0].features.length, 10);
testClient.drain(done);
});
});
});
describe('analysis-layers-dataviews-mvt', function () {
function createMapConfig(layers, dataviews, analysis) {

2
test/acceptance/ported/attributes.js
View File

@ -125,7 +125,7 @@ describe('attributes', function() {
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors);
var msg = parsed.errors[0];
assert.ok(msg.match(/0 features.*identified by fid -666/), msg);
assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1");
return null;
},
function finish(err) {

84
test/support/test-client.js
View File

@ -1253,3 +1253,87 @@ TestClient.prototype.getAnalysesCatalog = function (params, callback) {
}
);
};
TestClient.prototype.getNamedTile = function (name, z, x, y, format, options, callback) {
const { params } = options;
if (!this.apiKey) {
return callback(new Error('apiKey param is mandatory to create a new template'));
}
const createTemplateRequest = {
url: `/api/v1/map/named?${qs.stringify({ api_key: this.apiKey })}`,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(this.template)
};
const createTemplateResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
assert.response(this.server, createTemplateRequest, createTemplateResponse, (res, err) => {
if (err) {
return callback(err);
}
const templateId = JSON.parse(res.body).template_id;
const queryParams = params ? `?${qs.stringify(params)}` : '';
const url = `/api/v1/map/named/${templateId}/all/${[z,x,y].join('/')}.${format}${queryParams}`;
const namedTileRequest = {
url,
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
};
let contentType;
switch (format) {
case 'png':
contentType = 'image/png';
break;
case 'mvt':
contentType = 'application/x-protobuf';
break;
default:
contentType = 'application/json';
break;
}
const namedTileResponse = Object.assign({
status: 200,
headers: {
'content-type': contentType
}
}, options.response);
assert.response(this.server, namedTileRequest, namedTileResponse, (res, err) => {
let body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/x-protobuf':
body = new mapnik.VectorTile(z, x, y);
body.setDataSync(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body;
break;
}
return callback(err, res, body);
});
});
};

7
test/unit/cartodb/lzmaMiddleware.test.js
View File

@ -12,6 +12,7 @@ describe('lzma-middleware', function() {
}
};
testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
const lzma = lzmaMiddleware();
var req = {
headers: {
host:'localhost'
@ -19,9 +20,13 @@ describe('lzma-middleware', function() {
query: {
api_key: 'test',
lzma: data
},
profiler: {
done: function () {}
}
};
lzmaMiddleware(req, {}, function(err) {
lzma(req, {}, function(err) {
if ( err ) {
return done(err);
}

90
test/unit/cartodb/ported/tile_stats.test.js
View File

@ -1,90 +0,0 @@
require('../../../support/test_helper.js');
var assert = require('assert');
var LayergroupController = require('../../../../lib/cartodb/controllers/layergroup');
describe('tile stats', function() {
beforeEach(function () {
this.statsClient = global.statsClient;
});
afterEach(function() {
global.statsClient = this.statsClient;
});
it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() {
var expectedCalls = 2, // it will call increment once for the general error
invalidFormat = 'png2',
invalidFormatRegexp = new RegExp('invalid'),
formatMatched = false;
mockStatsClientGetInstance({
increment: function(label) {
formatMatched = formatMatched || !!label.match(invalidFormatRegexp);
expectedCalls--;
}
});
var layergroupController = new LayergroupController();
var reqMock = {
profiler: { toJSONString:function() {} },
params: {
format: invalidFormat
}
};
var resMock = {
status: function() { return this; },
set: function() {},
json: function() {},
jsonp: function() {},
send: function() {}
};
var next = function () {};
layergroupController.finalizeGetTileOrGrid('Unsupported format png2', reqMock, resMock, null, null, next);
assert.ok(formatMatched, 'Format was never matched in increment method');
assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method');
});
it('finalizeGetTileOrGrid calls statsClient when format is supported', function() {
var expectedCalls = 2, // general error + format error
validFormat = 'png',
validFormatRegexp = new RegExp(validFormat),
formatMatched = false;
mockStatsClientGetInstance({
increment: function(label) {
formatMatched = formatMatched || !!label.match(validFormatRegexp);
expectedCalls--;
}
});
var reqMock = {
profiler: { toJSONString:function() {} },
params: {
format: validFormat
}
};
var resMock = {
status: function() { return this; },
set: function() {},
json: function() {},
jsonp: function() {},
send: function() {}
};
var layergroupController = new LayergroupController();
var next = function () {};
layergroupController.finalizeGetTileOrGrid('Another error happened', reqMock, resMock, null, null, next);
assert.ok(formatMatched, 'Format was never matched in increment method');
assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method');
});
function mockStatsClientGetInstance(instance) {
global.statsClient = Object.assign(global.statsClient, instance);
}
});

17
test/unit/cartodb/prepare-context.test.js
View File

@ -10,7 +10,7 @@ var TemplateMaps = require('../../../lib/cartodb/backends/template_maps');
const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/context/clean-up-query-params');
const authorizeMiddleware = require('../../../lib/cartodb/middleware/context/authorize');
const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/context/db-conn-setup');
const apikeyCredentialsMiddleware = require('../../../lib/cartodb/middleware/context/apikey-credentials');
const credentialsMiddleware = require('../../../lib/cartodb/middleware/context/credentials');
const localsMiddleware = require('../../../lib/cartodb/middleware/context/locals');
var windshaft = require('windshaft');
@ -24,7 +24,7 @@ describe('prepare-context', function() {
let cleanUpQueryParams;
let dbConnSetup;
let authorize;
let setApikeyCredentials;
let setCredentials;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
@ -37,7 +37,7 @@ describe('prepare-context', function() {
cleanUpQueryParams = cleanUpQueryParamsMiddleware();
authorize = authorizeMiddleware(authApi);
dbConnSetup = dbConnSetupMiddleware(pgConnection);
setApikeyCredentials = apikeyCredentialsMiddleware();
setCredentials = credentialsMiddleware();
});
@ -61,14 +61,17 @@ describe('prepare-context', function() {
}
res.locals.user = 'localhost';
res.set = function () {};
return res;
}
it('res.locals are created', function(done) {
const locals = localsMiddleware();
let req = {};
let res = {};
localsMiddleware(prepareRequest(req), prepareResponse(res), function(err) {
locals(prepareRequest(req), prepareResponse(res), function(err) {
if ( err ) { done(err); return; }
assert.ok(res.hasOwnProperty('locals'), 'response has locals');
done();
@ -194,7 +197,7 @@ describe('prepare-context', function() {
}
};
var res = {};
setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) {
setCredentials(prepareRequest(req), prepareResponse(res), function (err) {
if (err) {
return done(err);
}
@ -215,7 +218,7 @@ describe('prepare-context', function() {
}
};
var res = {};
setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) {
setCredentials(prepareRequest(req), prepareResponse(res), function (err) {
if (err) {
return done(err);
}
@ -234,7 +237,7 @@ describe('prepare-context', function() {
}
};
var res = {};
setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) {
setCredentials(prepareRequest(req), prepareResponse(res), function (err) {
if (err) {
return done(err);
}

69
yarn.lock
View File

@ -11,11 +11,11 @@
node-pre-gyp "~0.6.30"
protozero "1.5.1"
"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb1":
version "2.5.1-cdb1"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/b0b5559f948e77b337bc9a9ae0bf6ec4249fba21"
"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb3":
version "2.5.1-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/e61c7752c033595a273dcd1d4b267252b174bd28"
dependencies:
"@carto/mapnik" "~3.6.2-carto.0"
"@carto/mapnik" "3.6.2-carto.2"
"@mapbox/sphericalmercator" "~1.0.1"
mapnik-pool "~0.1.3"
@ -23,7 +23,7 @@
version "1.0.5"
resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2"
abaculus@cartodb/abaculus#2.0.3-cdb2:
"abaculus@github:cartodb/abaculus#2.0.3-cdb2":
version "2.0.3-cdb2"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc"
dependencies:
@ -238,9 +238,9 @@ camelcase@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
camshaft@0.61.2:
version "0.61.2"
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.2.tgz#5c0d43ca769377c6cfb9808f1023d20ea3df55b3"
camshaft@0.61.3:
version "0.61.3"
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.3.tgz#a1507f19e39dbf98940298ba1704b742741a5053"
dependencies:
async "^1.5.2"
bunyan "1.8.1"
@ -249,7 +249,7 @@ camshaft@0.61.2:
dot "^1.0.3"
request "^2.69.0"
canvas@cartodb/node-canvas#1.6.2-cdb2:
"canvas@github:cartodb/node-canvas#1.6.2-cdb2":
version "1.6.2-cdb2"
resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
dependencies:
@ -275,7 +275,7 @@ carto@CartoDB/carto#0.15.1-cdb1:
optimist "~0.6.0"
underscore "~1.6.0"
carto@cartodb/carto#0.15.1-cdb3:
"carto@github:cartodb/carto#0.15.1-cdb3":
version "0.15.1-cdb3"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
dependencies:
@ -1885,7 +1885,34 @@ request@2.81.0:
tunnel-agent "^0.6.0"
uuid "^3.0.0"
request@2.x, request@^2.55.0, request@^2.69.0, request@^2.83.0:
request@2.85.0:
version "2.85.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa"
dependencies:
aws-sign2 "~0.7.0"
aws4 "^1.6.0"
caseless "~0.12.0"
combined-stream "~1.0.5"
extend "~3.0.1"
forever-agent "~0.6.1"
form-data "~2.3.1"
har-validator "~5.0.3"
hawk "~6.0.2"
http-signature "~1.2.0"
is-typedarray "~1.0.0"
isstream "~0.1.2"
json-stringify-safe "~5.0.1"
mime-types "~2.1.17"
oauth-sign "~0.8.2"
performance-now "^2.1.0"
qs "~6.5.1"
safe-buffer "^5.1.1"
stringstream "~0.0.5"
tough-cookie "~2.3.3"
tunnel-agent "^0.6.0"
uuid "^3.1.0"
request@2.x, request@^2.55.0, request@^2.69.0:
version "2.83.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356"
dependencies:
@ -2212,11 +2239,11 @@ through@2:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb5:
version "0.6.18-cdb5"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cec846025e60837c60af193d600d972917ea8d35"
"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb7":
version "0.6.18-cdb7"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/488d2acd65c89cc5382d996eabe8dc1f5051ce0f"
dependencies:
"@carto/mapnik" "~3.6.2-carto.0"
"@carto/mapnik" "3.6.2-carto.2"
generic-pool "~2.4.0"
mime "~1.6.0"
sphericalmercator "~1.0.4"
@ -2373,12 +2400,12 @@ window-size@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@4.5.3:
version "4.5.3"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.3.tgz#45e792af06b224f78f44b6eb3b0ecb9d90dcc943"
windshaft@4.5.6:
version "4.5.6"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.6.tgz#566c635f25e4dfc18b46cffe1b22b8c9626f8295"
dependencies:
"@carto/mapnik" "3.6.2-carto.2"
"@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb1
"@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb3
abaculus cartodb/abaculus#2.0.3-cdb2
canvas cartodb/node-canvas#1.6.2-cdb2
carto cartodb/carto#0.15.1-cdb3
@ -2388,12 +2415,12 @@ windshaft@4.5.3:
grainstore "1.8.2"
queue-async "~1.0.7"
redis-mpool "^0.5.0"
request "^2.83.0"
request "2.85.0"
semver "~5.0.3"
sphericalmercator "1.0.4"
step "~0.0.6"
tilelive "5.12.2"
tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb5
tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb7
torque.js "~2.11.0"
underscore "~1.6.0"