diff --git a/NEWS.md b/NEWS.md index 712aacd9..9be83319 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,9 +2,10 @@ ## 5.4.0 Released yyyy-mm-dd - - Upgrades Windshaft to 4.5.3 + - Upgrades Windshaft to 4.5.4 ([Mapnik top metrics](https://github.com/CartoDB/Windshaft/pull/597), [AttributesBackend allows multiple features if all the attributes are the same](https://github.com/CartoDB/Windshaft/pull/602)) - Implemented middleware to authorize users via new Api Key system - Keep the old authorization system as fallback + - Aggregation widget: Remove NULL categories in 'count' aggregations too ## 5.3.1 Released 2018-02-13 diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js index 8782f907..e9f10262 100644 --- a/lib/cartodb/api/auth_api.js +++ b/lib/cartodb/api/auth_api.js @@ -62,7 +62,7 @@ function isValidApiKey(apikey) { // AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) { const apikeyToken = res.locals.api_key; - const apikeyUsername = res.locals.apikeyUsername; + const basicAuthUsername = res.locals.basicAuthUsername; if ( ! apikeyToken ) { return callback(null, false); // no api key, no authorization... @@ -91,7 +91,7 @@ AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) { return callback(error); } - if (!usernameMatches(apikeyUsername, res.locals.user)) { + if (!usernameMatches(basicAuthUsername, res.locals.user)) { const error = new Error('Forbidden'); error.type = 'auth'; error.subtype = 'api-key-username-mismatch'; @@ -149,8 +149,8 @@ function isNameNotFoundError (err) { return err.message && -1 !== err.message.indexOf('name not found'); } -function usernameMatches (apikeyUsername, requestUsername) { - return !(apikeyUsername && (apikeyUsername !== requestUsername)); +function usernameMatches (basicAuthUsername, requestUsername) { + return !(basicAuthUsername && (basicAuthUsername !== requestUsername)); } /** diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index fb75c633..db3550c8 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -12,7 +12,7 @@ AnalysesController.prototype.register = function (app) { app.get( `${app.base_url_mapconfig}/analyses/catalog`, cors(), - userMiddleware, + userMiddleware(), this.prepareContext, this.createPGClient(), this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index d9251dd0..182fcff5 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -1,20 +1,22 @@ -var assert = require('assert'); -var step = require('step'); - -var cors = require('../middleware/cors'); -var userMiddleware = require('../middleware/user'); -var allowQueryParams = require('../middleware/allow-query-params'); -var vectorError = require('../middleware/vector-error'); - -var DataviewBackend = require('../backends/dataview'); -var AnalysisStatusBackend = require('../backends/analysis-status'); - -var MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); - -var QueryTables = require('cartodb-query-tables'); +const cors = require('../middleware/cors'); +const userMiddleware = require('../middleware/user'); +const allowQueryParams = require('../middleware/allow-query-params'); +const vectorError = require('../middleware/vector-error'); +const DataviewBackend = require('../backends/dataview'); +const AnalysisStatusBackend = require('../backends/analysis-status'); +const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); +const QueryTables = require('cartodb-query-tables'); +const SUPPORTED_FORMATS = { + grid_json: true, + json_torque: true, + torque_json: true, + png: true, + png32: true, + mvt: true +}; /** - * @param {AuthApi} authApi + * @param {prepareContext} prepareContext * @param {PgConnection} pgConnection * @param {MapStore} mapStore * @param {TileBackend} tileBackend @@ -46,64 +48,119 @@ function LayergroupController(prepareContext, pgConnection, mapStore, tileBacken module.exports = LayergroupController; LayergroupController.prototype.register = function(app) { + const { base_url_mapconfig: basePath } = app; + app.get( - app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format', + `${basePath}/:token/:z/:x/:y@:scale_factor?x.:format`, cors(), - userMiddleware, + userMiddleware(), this.prepareContext, - this.tile.bind(this), + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + getTile(this.tileBackend, 'map_tile'), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + incrementSuccessMetrics(global.statsClient), + sendResponse(), + incrementErrorMetrics(global.statsClient), + tileError(), vectorError() ); app.get( - app.base_url_mapconfig + '/:token/:z/:x/:y.:format', + `${basePath}/:token/:z/:x/:y.:format`, cors(), - userMiddleware, + userMiddleware(), this.prepareContext, - this.tile.bind(this), + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + getTile(this.tileBackend, 'map_tile'), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + incrementSuccessMetrics(global.statsClient), + sendResponse(), + incrementErrorMetrics(global.statsClient), + tileError(), vectorError() ); app.get( - app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)', + `${basePath}/:token/:layer/:z/:x/:y.(:format)`, + distinguishLayergroupFromStaticRoute(), cors(), - userMiddleware, - validateLayerRouteMiddleware, + userMiddleware(), this.prepareContext, - this.layer.bind(this), + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + getTile(this.tileBackend, 'maplayer_tile'), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + incrementSuccessMetrics(global.statsClient), + sendResponse(), + incrementErrorMetrics(global.statsClient), + tileError(), vectorError() ); app.get( - app.base_url_mapconfig + '/:token/:layer/attributes/:fid', + `${basePath}/:token/:layer/attributes/:fid`, cors(), - userMiddleware, + userMiddleware(), this.prepareContext, - this.attributes.bind(this) + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + getFeatureAttributes(this.attributesBackend), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + sendResponse() ); + const forcedFormat = 'png'; + app.get( - app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format', + `${basePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, cors(), - userMiddleware, + userMiddleware(), allowQueryParams(['layer']), this.prepareContext, - this.center.bind(this) + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), + getPreviewImageByCenter(this.previewBackend), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + sendResponse() ); app.get( - app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', + `${basePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, cors(), - userMiddleware, + userMiddleware(), allowQueryParams(['layer']), this.prepareContext, - this.bbox.bind(this) + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), + getPreviewImageByBoundingBox(this.previewBackend), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + sendResponse() ); // Undocumented/non-supported API endpoint methods. // Use at your own peril. - var allowedDataviewQueryParams = [ + const allowedDataviewQueryParams = [ 'filters', // json 'own_filter', // 0, 1 'no_filters', // 0, 1 @@ -119,395 +176,459 @@ LayergroupController.prototype.register = function(app) { ]; app.get( - app.base_url_mapconfig + '/:token/dataview/:dataviewName', + `${basePath}/:token/dataview/:dataviewName`, cors(), - userMiddleware, + userMiddleware(), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, - this.dataview.bind(this) + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + getDataview(this.dataviewBackend), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + sendResponse() ); app.get( - app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName', + `${basePath}/:token/:layer/widget/:dataviewName`, cors(), - userMiddleware, + userMiddleware(), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, - this.dataview.bind(this) + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + getDataview(this.dataviewBackend), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + sendResponse() ); app.get( - app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', + `${basePath}/:token/dataview/:dataviewName/search`, cors(), - userMiddleware, + userMiddleware(), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, - this.dataviewSearch.bind(this) + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + dataviewSearch(this.dataviewBackend), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + sendResponse() ); app.get( - app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search', + `${basePath}/:token/:layer/widget/:dataviewName/search`, cors(), - userMiddleware, + userMiddleware(), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, - this.dataviewSearch.bind(this) + createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), + dataviewSearch(this.dataviewBackend), + setCacheControlHeader(), + setLastModifiedHeader(), + getAffectedTables(this.layergroupAffectedTables, this.pgConnection), + setCacheChannelHeader(), + setSurrogateKeyHeader(this.surrogateKeysCache), + sendResponse() ); app.get( - app.base_url_mapconfig + '/:token/analysis/node/:nodeId', + `${basePath}/:token/analysis/node/:nodeId`, cors(), - userMiddleware, + userMiddleware(), this.prepareContext, - this.analysisNodeStatus.bind(this) + analysisNodeStatus(this.analysisStatusBackend), + sendResponse() ); }; -LayergroupController.prototype.analysisNodeStatus = function(req, res, next) { - var self = this; +function distinguishLayergroupFromStaticRoute () { + return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) { + if (req.params.token === 'static') { + return next('route'); + } - step( - function retrieveNodeStatus() { - self.analysisStatusBackend.getNodeStatus(res.locals, this); - }, - function finish(err, nodeStatus, stats) { - req.profiler.add(stats || {}); + next(); + }; +} + +function analysisNodeStatus (analysisStatusBackend) { + return function analysisNodeStatusMiddleware(req, res, next) { + analysisStatusBackend.getNodeStatus(res.locals, (err, nodeStatus, stats = {}) => { + req.profiler.add(stats); if (err) { err.label = 'GET NODE STATUS'; - next(err); - } else { - self.sendResponse(req, res, nodeStatus, 200, { - 'Cache-Control': 'public,max-age=5', - 'Last-Modified': new Date().toUTCString() - }); + return next(err); } + + res.set({ + 'Cache-Control': 'public,max-age=5', + 'Last-Modified': new Date().toUTCString() + }); + + res.body = nodeStatus; + + next(); + }); + }; +} + +function getRequestParams(locals) { + const params = Object.assign({}, locals); + + delete params.mapConfigProvider; + delete params.allowedQueryParams; + + return params; +} + +function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) { + return function createMapStoreMapConfigProviderMiddleware (req, res, next) { + const { user } = res.locals; + + const params = getRequestParams(res.locals); + + if (forcedFormat) { + params.format = forcedFormat; + params.layer = params.layer || 'all'; } - ); -}; -LayergroupController.prototype.dataview = function(req, res, next) { - var self = this; + const mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params); - step( - function retrieveDataview() { - var mapConfigProvider = new MapStoreMapConfigProvider( - self.mapStore, res.locals.user, self.userLimitsApi, res.locals - ); - self.dataviewBackend.getDataview( - mapConfigProvider, - res.locals.user, - res.locals, - this - ); - }, - function finish(err, dataview, stats) { - req.profiler.add(stats || {}); + mapConfigProvider.getMapConfig((err, mapconfig) => { + if (err) { + return next(err); + } + + res.locals.mapConfigProvider = mapConfigProvider; + res.locals.mapconfig = mapconfig; + + next(); + }); + }; +} + +function getDataview (dataviewBackend) { + return function getDataviewMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const params = getRequestParams(res.locals); + + dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { + req.profiler.add(stats); if (err) { err.label = 'GET DATAVIEW'; - next(err); - } else { - self.sendResponse(req, res, dataview, 200); + return next(err); } - } - ); -}; -LayergroupController.prototype.dataviewSearch = function(req, res, next) { - var self = this; + res.body = dataview; - step( - function searchDataview() { - var mapConfigProvider = new MapStoreMapConfigProvider( - self.mapStore, res.locals.user, self.userLimitsApi, res.locals - ); - self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this); - }, - function finish(err, searchResult, stats) { - req.profiler.add(stats || {}); + next(); + }); + }; +} + +function dataviewSearch (dataviewBackend) { + return function dataviewSearchMiddleware (req, res, next) { + const { user, dataviewName, mapConfigProvider } = res.locals; + const params = getRequestParams(res.locals); + + dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { + req.profiler.add(stats); if (err) { err.label = 'GET DATAVIEW SEARCH'; - next(err); - } else { - self.sendResponse(req, res, searchResult, 200); + return next(err); } - } - ); -}; + res.body = searchResult; -LayergroupController.prototype.attributes = function(req, res, next) { - var self = this; + next(); + }); + }; +} - req.profiler.start('windshaft.maplayer_attribute'); +function getFeatureAttributes (attributesBackend) { + return function getFeatureAttributesMiddleware (req, res, next) { + req.profiler.start('windshaft.maplayer_attribute'); - step( - function retrieveFeatureAttributes() { - var mapConfigProvider = new MapStoreMapConfigProvider( - self.mapStore, res.locals.user, self.userLimitsApi, res.locals - ); - self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this); - }, - function finish(err, tile, stats) { - req.profiler.add(stats || {}); + const { mapConfigProvider } = res.locals; + const params = getRequestParams(res.locals); + + attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => { + req.profiler.add(stats); if (err) { err.label = 'GET ATTRIBUTES'; - next(err); - } else { - self.sendResponse(req, res, tile, 200); + return next(err); } - } - ); -}; + res.body = tile; -// Gets a tile for a given token and set of tile ZXY coords. (OSM style) -LayergroupController.prototype.tile = function(req, res, next) { - req.profiler.start('windshaft.map_tile'); - this.tileOrLayer(req, res, next); -}; - -// Gets a tile for a given token, layer set of tile ZXY coords. (OSM style) -LayergroupController.prototype.layer = function(req, res, next) { - req.profiler.start('windshaft.maplayer_tile'); - this.tileOrLayer(req, res, next); -}; - -LayergroupController.prototype.tileOrLayer = function (req, res, next) { - var self = this; - - step( - function mapController$getTileOrGrid() { - self.tileBackend.getTile( - new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals), - res.locals, this - ); - }, - function mapController$finalize(err, tile, headers, stats) { - req.profiler.add(stats); - self.finalizeGetTileOrGrid(err, req, res, tile, headers, next); - } - ); -}; - -function getStatusCode(tile, format){ - return tile.length===0 && format==='mvt'? 204:200; + next(); + }); + }; } -// This function is meant for being called as the very last -// step by all endpoints serving tiles or grids -LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) { - var supportedFormats = { - grid_json: true, - json_torque: true, - torque_json: true, - png: true, - png32: true, - mvt: true - }; +function getStatusCode(tile, format){ + return tile.length === 0 && format === 'mvt'? 204 : 200; +} - var formatStat = 'invalid'; - if (req.params.format) { - var format = req.params.format.replace('.', '_'); - if (supportedFormats[format]) { - formatStat = format; - } - } +function parseFormat (format = '') { + const prettyFormat = format.replace('.', '_'); + return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; +} - if (err) { - // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 - var errMsg = err.message ? ( '' + err.message ) : ( '' + err ); +function getTile (tileBackend, profileLabel = 'tile') { + return function getTileMiddleware (req, res, next) { + req.profiler.start(`windshaft.${profileLabel}`); - // Rewrite mapnik parsing errors to start with layer number - var matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); - if (matches) { - errMsg = 'style'+matches[2]+': ' + matches[1]; - } - err.message = errMsg; + const { mapConfigProvider } = res.locals; + const params = getRequestParams(res.locals); - err.label = 'TILE RENDER'; - next(err); - - global.statsClient.increment('windshaft.tiles.error'); - global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); - } else { - this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers); - global.statsClient.increment('windshaft.tiles.success'); - global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); - } -}; - -LayergroupController.prototype.bbox = function(req, res, next) { - this.staticMap(req, res, +req.params.width, +req.params.height, { - west: +req.params.west, - north: +req.params.north, - east: +req.params.east, - south: +req.params.south - }, null, next); -}; - -LayergroupController.prototype.center = function(req, res, next) { - this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, { - lng: +req.params.lng, - lat: +req.params.lat - }, next); -}; - -LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) { - var format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - // We force always the tile to be generated using PNG because - // is the only format we support by now - res.locals.format = 'png'; - res.locals.layer = res.locals.layer || 'all'; - - var self = this; - - step( - function getImage() { - if (center) { - self.previewBackend.getImage( - new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals), - format, width, height, zoom, center, this); - } else { - self.previewBackend.getImage( - new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals), - format, width, height, zoom /* bounds */, this); - } - }, - function handleImage(err, image, headers, stats) { - req.profiler.done('render-' + format); - req.profiler.add(stats || {}); + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { + req.profiler.add(stats); if (err) { - err.label = 'STATIC_MAP'; - next(err); - } else { - res.set('Content-Type', headers['Content-Type'] || 'image/' + format); - self.sendResponse(req, res, image, 200); - } - } - ); -}; - -LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) { - var self = this; - - req.profiler.done('res'); - - res.set('Cache-Control', 'public,max-age=31536000'); - - // Set Last-Modified header - var lastUpdated; - if (res.locals.cache_buster) { - // Assuming cache_buster is a timestamp - lastUpdated = new Date(parseInt(res.locals.cache_buster)); - } else { - lastUpdated = new Date(); - } - res.set('Last-Modified', lastUpdated.toUTCString()); - - var dbName = res.locals.dbname; - step( - function getAffectedTables() { - self.getAffectedTables(res.locals.user, dbName, res.locals.token, this); - }, - function sendResponse(err, affectedTables) { - req.profiler.done('affectedTables'); - if (err) { - global.logger.warn('ERROR generating cache channel: ' + err); - } - if (!!affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - self.surrogateKeysCache.tag(res, affectedTables); + return next(err); } if (headers) { res.set(headers); } - res.status(status); + const formatStat = parseFormat(req.params.format); - if (!Buffer.isBuffer(body) && typeof body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(body); - } else { - res.json(body); - } - } else { - res.send(body); - } - } - ); -}; + res.statusCode = getStatusCode(tile, formatStat); + res.body = tile; -LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) { - - if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) { - return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId)); - } - - var self = this; - step( - function extractSQL() { - step( - function loadFromStore() { - self.mapStore.load(layergroupId, this); - }, - function getSQL(err, mapConfig) { - assert.ifError(err); - - var queries = []; - mapConfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - queries.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); - - return queries.length ? queries.join(';') : null; - }, - this - ); - }, - function findAffectedTables(err, sql) { - assert.ifError(err); - - if ( ! sql ) { - throw new Error("this request doesn't need an X-Cache-Channel generated"); - } - - step( - function getConnection() { - self.pgConnection.getConnection(user, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function buildCacheChannel(err, tables) { - assert.ifError(err); - self.layergroupAffectedTables.set(dbName, layergroupId, tables); - - return tables; - }, - callback - ); -}; - - -function validateLayerRouteMiddleware(req, res, next) { - if (req.params.token === 'static') { - return next('route'); - } - - next(); + next(); + }); + }; +} + +function getPreviewImageByCenter (previewBackend) { + return function getPreviewImageByCenterMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const zoom = +req.params.z; + const center = { + lng: +req.params.lng, + lat: +req.params.lat + }; + + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.body = image; + + next(); + }); + }; +} + +function getPreviewImageByBoundingBox (previewBackend) { + return function getPreviewImageByBoundingBoxMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const bounds = { + west: +req.params.west, + north: +req.params.north, + east: +req.params.east, + south: +req.params.south + }; + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.body = image; + + next(); + }); + }; +} + +function setLastModifiedHeader () { + return function setLastModifiedHeaderMiddleware (req, res, next) { + let { cache_buster: cacheBuster } = res.locals; + + cacheBuster = parseInt(cacheBuster, 10); + + const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date(); + + res.set('Last-Modified', lastUpdated.toUTCString()); + + next(); + }; +} + +function setCacheControlHeader () { + return function setCacheControlHeaderMiddleware (req, res, next) { + res.set('Cache-Control', 'public,max-age=31536000'); + + next(); + }; +} + +function getAffectedTables (layergroupAffectedTables, pgConnection) { + return function getAffectedTablesMiddleware (req, res, next) { + const { user, dbname, token, mapconfig } = res.locals; + + if (layergroupAffectedTables.hasAffectedTables(dbname, token)) { + res.locals.affectedTables = layergroupAffectedTables.get(dbname, token); + return next(); + } + + pgConnection.getConnection(user, (err, connection) => { + if (err) { + global.logger.warn('ERROR generating cache channel:', err); + return next(); + } + + const sql = []; + mapconfig.getLayers().forEach(function(layer) { + sql.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(function(table) { + sql.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { + req.profiler.done('getAffectedTablesFromQuery'); + if (err) { + global.logger.warn('ERROR generating cache channel: ', err); + return next(); + } + + // feed affected tables cache so it can be reused from, for instance, map controller + layergroupAffectedTables.set(dbname, token, affectedTables); + + res.locals.affectedTables = affectedTables; + + next(); + }); + }); + }; +} + +function setCacheChannelHeader () { + return function setCacheChannelHeaderMiddleware (req, res, next) { + const { affectedTables } = res.locals; + + if (affectedTables) { + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + } + + next(); + }; +} + +function setSurrogateKeyHeader (surrogateKeysCache) { + return function setSurrogateKeyHeaderMiddleware (req, res, next) { + const { affectedTables } = res.locals; + + if (affectedTables) { + surrogateKeysCache.tag(res, affectedTables); + } + + next(); + }; +} + +function incrementSuccessMetrics (statsClient) { + return function incrementSuccessMetricsMiddleware (req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.success'); + statsClient.increment(`windshaft.tiles.${formatStat}.success`); + + next(); + }; +} + +function sendResponse () { + return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); + + res.status(res.statusCode || 200); + + if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') { + if (req.query && req.query.callback) { + res.jsonp(res.body); + } else { + res.json(res.body); + } + } else { + res.send(res.body); + } + }; +} + +function incrementErrorMetrics (statsClient) { + return function incrementErrorMetricsMiddleware (err, req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.error'); + statsClient.increment(`windshaft.tiles.${formatStat}.error`); + + next(err); + }; +} + +function tileError () { + return function tileErrorMiddleware (err, req, res, next) { + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + + // Rewrite mapnik parsing errors to start with layer number + const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); + + if (matches) { + errMsg = `style${matches[2]}: ${matches[1]}`; + } + + err.message = errMsg; + err.label = 'TILE RENDER'; + + next(err); + }; } diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 34660ce0..a9077339 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -69,7 +69,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal return [ cors(), - userMiddleware, + userMiddleware(), allowQueryParams(['aggregation']), this.prepareContext, this.initProfiler(isTemplateInstantiation), diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 93566cc9..ffa007a1 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -48,7 +48,7 @@ NamedMapsController.prototype.register = function(app) { app.get( app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)', cors(), - userMiddleware, + userMiddleware(), this.prepareContext, this.getNamedMapProvider(tileOptions), this.getAffectedTables(), @@ -70,7 +70,7 @@ NamedMapsController.prototype.register = function(app) { app.get( app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', cors(), - userMiddleware, + userMiddleware(), allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), this.prepareContext, this.getNamedMapProvider(staticOptions), diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index b1c6a451..0728b38c 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -2,12 +2,7 @@ const { templateName } = require('../backends/template_maps'); const cors = require('../middleware/cors'); const userMiddleware = require('../middleware/user'); const localsMiddleware = require('../middleware/context/locals'); -const apikeyCredentialsMiddleware = require('../middleware/context/apikey-credentials'); - -const apikeyMiddleware = [ - localsMiddleware, - apikeyCredentialsMiddleware(), -]; +const credentialsMiddleware = require('../middleware/context/credentials'); /** * @param {AuthApi} authApi @@ -28,8 +23,9 @@ NamedMapsAdminController.prototype.register = function (app) { app.post( `${base_url_templated}/`, cors(), - userMiddleware, - apikeyMiddleware, + userMiddleware(), + localsMiddleware(), + credentialsMiddleware(), this.checkContentType('POST', 'POST TEMPLATE'), this.authorizedByAPIKey('create', 'POST TEMPLATE'), this.create() @@ -38,8 +34,9 @@ NamedMapsAdminController.prototype.register = function (app) { app.put( `${base_url_templated}/:template_id`, cors(), - userMiddleware, - apikeyMiddleware, + userMiddleware(), + localsMiddleware(), + credentialsMiddleware(), this.checkContentType('PUT', 'PUT TEMPLATE'), this.authorizedByAPIKey('update', 'PUT TEMPLATE'), this.update() @@ -48,8 +45,9 @@ NamedMapsAdminController.prototype.register = function (app) { app.get( `${base_url_templated}/:template_id`, cors(), - userMiddleware, - apikeyMiddleware, + userMiddleware(), + localsMiddleware(), + credentialsMiddleware(), this.authorizedByAPIKey('get', 'GET TEMPLATE'), this.retrieve() ); @@ -57,8 +55,9 @@ NamedMapsAdminController.prototype.register = function (app) { app.delete( `${base_url_templated}/:template_id`, cors(), - userMiddleware, - apikeyMiddleware, + userMiddleware(), + localsMiddleware(), + credentialsMiddleware(), this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'), this.destroy() ); @@ -66,8 +65,9 @@ NamedMapsAdminController.prototype.register = function (app) { app.get( `${base_url_templated}/`, cors(), - userMiddleware, - apikeyMiddleware, + userMiddleware(), + localsMiddleware(), + credentialsMiddleware(), this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'), this.list() ); diff --git a/lib/cartodb/middleware/allow-query-params.js b/lib/cartodb/middleware/allow-query-params.js index 7ec31d74..cf90e69b 100644 --- a/lib/cartodb/middleware/allow-query-params.js +++ b/lib/cartodb/middleware/allow-query-params.js @@ -1,8 +1,9 @@ -module.exports = function allowQueryParams(params) { +module.exports = function allowQueryParams (params) { if (!Array.isArray(params)) { throw new Error('allowQueryParams must receive an Array of params'); } - return function allowQueryParamsMiddleware(req, res, next) { + + return function allowQueryParamsMiddleware (req, res, next) { res.locals.allowedQueryParams = params; next(); }; diff --git a/lib/cartodb/middleware/context/authorize.js b/lib/cartodb/middleware/context/authorize.js index a42b5407..a1323fa9 100644 --- a/lib/cartodb/middleware/context/authorize.js +++ b/lib/cartodb/middleware/context/authorize.js @@ -1,9 +1,8 @@ -module.exports = function authorizeMiddleware (authApi) { - return function (req, res, next) { - req.profiler.done('req2params.setup'); - +module.exports = function authorize (authApi) { + return function authorizeMiddleware (req, res, next) { authApi.authorize(req, res, (err, authorized) => { req.profiler.done('authorize'); + if (err) { return next(err); } diff --git a/lib/cartodb/middleware/context/apikey-credentials.js b/lib/cartodb/middleware/context/credentials.js similarity index 86% rename from lib/cartodb/middleware/context/apikey-credentials.js rename to lib/cartodb/middleware/context/credentials.js index aa0477f6..b202de47 100644 --- a/lib/cartodb/middleware/context/apikey-credentials.js +++ b/lib/cartodb/middleware/context/credentials.js @@ -1,19 +1,17 @@ -'use strict'; +const basicAuth = require('basic-auth'); -module.exports = function apikeyToken () { - return function apikeyTokenMiddleware(req, res, next) { +module.exports = function credentials () { + return function credentialsMiddleware(req, res, next) { const apikeyCredentials = getApikeyCredentialsFromRequest(req); + res.locals.api_key = apikeyCredentials.token; - res.locals.apikeyUsername = apikeyCredentials.username; - res.set('vary', 'Authorization'); //Honor Authorization header when caching. + res.locals.basicAuthUsername = apikeyCredentials.username; + res.set('vary', 'Authorization'); //Honor Authorization header when caching. + return next(); }; }; -//-------------------------------------------------------------------------------- - -const basicAuth = require('basic-auth'); - function getApikeyCredentialsFromRequest(req) { let apikeyCredentials = { token: null, diff --git a/lib/cartodb/middleware/context/db-conn-setup.js b/lib/cartodb/middleware/context/db-conn-setup.js index 068d77c2..ce3f6ac0 100644 --- a/lib/cartodb/middleware/context/db-conn-setup.js +++ b/lib/cartodb/middleware/context/db-conn-setup.js @@ -1,14 +1,17 @@ const _ = require('underscore'); -module.exports = function dbConnSetupMiddleware(pgConnection) { - return function dbConnSetup(req, res, next) { - const user = res.locals.user; +module.exports = function dbConnSetup (pgConnection) { + return function dbConnSetupMiddleware (req, res, next) { + const { user } = res.locals; + pgConnection.setDBConn(user, res.locals, (err) => { + req.profiler.done('dbConnSetup'); + if (err) { if (err.message && -1 !== err.message.indexOf('name not found')) { err.http_status = 404; } - req.profiler.done('req2params'); + return next(err); } @@ -18,12 +21,10 @@ module.exports = function dbConnSetupMiddleware(pgConnection) { dbhost: global.environment.postgres.host, dbport: global.environment.postgres.port }); - + res.set('X-Served-By-DB-Host', res.locals.dbhost); - req.profiler.done('req2params'); - - next(null); + next(); }); }; }; diff --git a/lib/cartodb/middleware/context/index.js b/lib/cartodb/middleware/context/index.js index 8922739f..70465895 100644 --- a/lib/cartodb/middleware/context/index.js +++ b/lib/cartodb/middleware/context/index.js @@ -1,16 +1,16 @@ const locals = require('./locals'); const cleanUpQueryParams = require('./clean-up-query-params'); const layergroupToken = require('./layergroup-token'); -const apikeyCredentials = require('./apikey-credentials'); +const credentials = require('./credentials'); const authorize = require('./authorize'); const dbConnSetup = require('./db-conn-setup'); module.exports = function prepareContextMiddleware(authApi, pgConnection) { return [ - locals, + locals(), cleanUpQueryParams(), - layergroupToken, - apikeyCredentials(), + layergroupToken(), + credentials(), authorize(authApi), dbConnSetup(pgConnection) ]; diff --git a/lib/cartodb/middleware/context/layergroup-token.js b/lib/cartodb/middleware/context/layergroup-token.js index 026d0806..c4aac23f 100644 --- a/lib/cartodb/middleware/context/layergroup-token.js +++ b/lib/cartodb/middleware/context/layergroup-token.js @@ -1,32 +1,33 @@ -var LayergroupToken = require('../../models/layergroup-token'); - -module.exports = function layergroupTokenMiddleware(req, res, next) { - if (!res.locals.token) { - return next(); - } - - var user = res.locals.user; - - var layergroupToken = LayergroupToken.parse(res.locals.token); - res.locals.token = layergroupToken.token; - res.locals.cache_buster = layergroupToken.cacheBuster; - - if (layergroupToken.signer) { - res.locals.signer = layergroupToken.signer; - if (!res.locals.signer) { - res.locals.signer = user; - } else if (res.locals.signer !== user) { - var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`); - err.type = 'auth'; - err.http_status = 403; - if (req.query && req.query.callback) { - err.http_status = 200; - } - - req.profiler.done('req2params'); - return next(err); - } - } - - return next(); +const LayergroupToken = require('../../models/layergroup-token'); +const authErrorMessageTemplate = function (signer, user) { + return `Cannot use map signature of user "${signer}" on db of user "${user}"`; +}; + +module.exports = function layergroupToken () { + return function layergroupTokenMiddleware (req, res, next) { + if (!res.locals.token) { + return next(); + } + + const user = res.locals.user; + + const layergroupToken = LayergroupToken.parse(res.locals.token); + + res.locals.token = layergroupToken.token; + res.locals.cache_buster = layergroupToken.cacheBuster; + + if (layergroupToken.signer) { + res.locals.signer = layergroupToken.signer; + + if (res.locals.signer !== user) { + const err = new Error(authErrorMessageTemplate(res.locals.signer, user)); + err.type = 'auth'; + err.http_status = (req.query && req.query.callback) ? 200: 403; + + return next(err); + } + } + + return next(); + }; }; diff --git a/lib/cartodb/middleware/context/locals.js b/lib/cartodb/middleware/context/locals.js index 0fdcce50..f6f70923 100644 --- a/lib/cartodb/middleware/context/locals.js +++ b/lib/cartodb/middleware/context/locals.js @@ -1,6 +1,7 @@ -module.exports = function localsMiddleware(req, res, next) { - // save req.params in res.locals - res.locals = Object.assign(req.params || {}, res.locals); +module.exports = function locals () { + return function localsMiddleware (req, res, next) { + res.locals = Object.assign(req.params || {}, res.locals); - next(); + next(); + }; }; diff --git a/lib/cartodb/middleware/cors.js b/lib/cartodb/middleware/cors.js index 227bb477..65b7cf4f 100644 --- a/lib/cartodb/middleware/cors.js +++ b/lib/cartodb/middleware/cors.js @@ -1,11 +1,14 @@ -module.exports = function cors(extraHeaders) { - return function(req, res, next) { - var baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token"; +module.exports = function cors (extraHeaders) { + return function corsMiddleware (req, res, next) { + let baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token"; + if(extraHeaders) { baseHeaders += ", " + extraHeaders; } + res.set("Access-Control-Allow-Origin", "*"); res.set("Access-Control-Allow-Headers", baseHeaders); + next(); }; }; diff --git a/lib/cartodb/middleware/lzma.js b/lib/cartodb/middleware/lzma.js index 6655cdeb..b0a94412 100644 --- a/lib/cartodb/middleware/lzma.js +++ b/lib/cartodb/middleware/lzma.js @@ -1,30 +1,33 @@ -'use strict'; - const LZMA = require('lzma').LZMA; -const lzmaWorker = new LZMA(); +module.exports = function lzma () { + const lzmaWorker = new LZMA(); -module.exports = function lzmaMiddleware(req, res, next) { - if (!req.query.hasOwnProperty('lzma')) { - return next(); - } - - // Decode (from base64) - var lzma = new Buffer(req.query.lzma, 'base64') - .toString('binary') - .split('') - .map(function(c) { - return c.charCodeAt(0) - 128; - }); - - // Decompress - lzmaWorker.decompress(lzma, function(result) { - try { - delete req.query.lzma; - Object.assign(req.query, JSON.parse(result)); - next(); - } catch (err) { - next(new Error('Error parsing lzma as JSON: ' + err)); + return function lzmaMiddleware (req, res, next) { + if (!req.query.hasOwnProperty('lzma')) { + return next(); } - }); + + // Decode (from base64) + var lzma = new Buffer(req.query.lzma, 'base64') + .toString('binary') + .split('') + .map(function(c) { + return c.charCodeAt(0) - 128; + }); + + // Decompress + lzmaWorker.decompress(lzma, function(result) { + try { + delete req.query.lzma; + Object.assign(req.query, JSON.parse(result)); + + req.profiler.done('lzma'); + + next(); + } catch (err) { + next(new Error('Error parsing lzma as JSON: ' + err)); + } + }); + }; }; diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/middleware/stats.js index 489ee645..83ff3054 100644 --- a/lib/cartodb/middleware/stats.js +++ b/lib/cartodb/middleware/stats.js @@ -2,10 +2,10 @@ const Profiler = require('../stats/profiler_proxy'); const debug = require('debug')('windshaft:cartodb:stats'); const onHeaders = require('on-headers'); -module.exports = function statsMiddleware(options) { +module.exports = function stats (options) { const { enabled = true, statsClient } = options; - return function stats(req, res, next) { + return function statsMiddleware (req, res, next) { req.profiler = new Profiler({ statsd_client: statsClient, profile: enabled diff --git a/lib/cartodb/middleware/user.js b/lib/cartodb/middleware/user.js index adf06203..9c7968bc 100644 --- a/lib/cartodb/middleware/user.js +++ b/lib/cartodb/middleware/user.js @@ -1,8 +1,11 @@ -var CdbRequest = require('../models/cdb_request'); -var cdbRequest = new CdbRequest(); +const CdbRequest = require('../models/cdb_request'); -module.exports = function userMiddleware(req, res, next) { - res.locals.user = cdbRequest.userByReq(req); +module.exports = function user () { + const cdbRequest = new CdbRequest(); - next(); + return function userMiddleware(req, res, next) { + res.locals.user = cdbRequest.userByReq(req); + + next(); + }; }; diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index f42f1c87..75e93b0a 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -1,5 +1,4 @@ const fs = require('fs'); - const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt'); module.exports = function vectorError() { diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 568d3cc2..7820ddbc 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -42,7 +42,7 @@ const rankedCategoriesQueryTpl = ctx => ` ${ctx.aggregationFn} AS value, row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank FROM (${filteredQueryTpl(ctx)}) filtered_source - ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''} + WHERE ${ctx.aggregation === "count" ? `${ctx.column}` : `${ctx.aggregationColumn}`} IS NOT NULL GROUP BY ${ctx.column} ORDER BY 2 DESC ) @@ -279,7 +279,7 @@ module.exports = class Aggregation extends BaseDataview { max_val = 0, categories_count = 0 } = result.rows[0] || {}; - + return { aggregation: this.aggregation, count: count, @@ -290,10 +290,10 @@ module.exports = class Aggregation extends BaseDataview { max: max_val, categoriesCount: categories_count, categories: result.rows.map(({ category, value, agg }) => { - return { + return { category: agg ? 'Other' : category, - value, - agg + value, + agg }; }) }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 46ff7122..532d155b 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -377,7 +377,7 @@ function bootstrap(opts) { statsClient: global.statsClient })); - app.use(lzmaMiddleware); + app.use(lzmaMiddleware()); // temporary measure until we upgrade to newer version expressjs so we can check err.status app.use(function(err, req, res, next) { diff --git a/package.json b/package.json index 8b87d90a..9200354d 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.5.3", + "windshaft": "4.5.4", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index e29d9ff4..204fcac7 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -70,12 +70,8 @@ describe('aggregations happy cases', function() { ].join(' UNION ALL '); operations.forEach(function (operation) { - var not = operation === 'count' ? ' not ' : ' '; - var description = 'should' + - not + - 'handle NULL values in category and aggregation columns using "' + - operation + - '" as aggregation operation'; + var description = 'should handle NULL values in category and aggregation columns using "' + + operation + '" as aggregation operation'; it(description, function (done) { this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val')); @@ -96,12 +92,7 @@ describe('aggregations happy cases', function() { } }); - if (operation === 'count') { - assert.ok(hasNullCategory, 'aggregation has not a category NULL'); - } else { - assert.ok(!hasNullCategory, 'aggregation has category NULL'); - } - + assert.ok(!hasNullCategory, 'aggregation has category NULL'); done(); }); }); @@ -425,3 +416,79 @@ describe('aggregation dataview tuned by categories query param', function () { }); }); }); + + + +describe('Count aggregation', function () { + const mapConfig = { + version: '1.5.0', + layers: [ + { + type: "cartodb", + options: { + source: { + "id": "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + dataviews: { + categories: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'count' + } + } + }, + analyses: [ + { + id: "a0", + type: "source", + params: { + query: ` + SELECT + null::geometry the_geom_webmercator, + CASE + WHEN x % 4 = 0 THEN 1 + WHEN x % 4 = 1 THEN 2 + WHEN x % 4 = 2 THEN 3 + ELSE null + END AS val, + CASE + WHEN x % 4 = 0 THEN 'category_1' + WHEN x % 4 = 1 THEN 'category_2' + WHEN x % 4 = 2 THEN 'category_3' + ELSE null + END AS cat + FROM generate_series(1, 1000) x + ` + } + } + ] + }; + + it(`should handle null values correctly when aggregationColumn isn't provided`, function (done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => { + assert.ifError(err); + assert.equal(dataview.categories.length, 3); + this.testClient.drain(done); + }); + }); + + it(`should handle null values correctly when aggregationColumn is provided`, function (done) { + mapConfig.dataviews.categories.options.aggregationColumn = 'val'; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => { + assert.ifError(err); + assert.equal(dataview.categories.length, 3); + this.testClient.drain(done); + }); + }); +}); \ No newline at end of file diff --git a/test/acceptance/ported/attributes.js b/test/acceptance/ported/attributes.js index abd4e5ab..f19fb8e8 100644 --- a/test/acceptance/ported/attributes.js +++ b/test/acceptance/ported/attributes.js @@ -125,7 +125,7 @@ describe('attributes', function() { var parsed = JSON.parse(res.body); assert.ok(parsed.errors); var msg = parsed.errors[0]; - assert.ok(msg.match(/0 features.*identified by fid -666/), msg); + assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1"); return null; }, function finish(err) { diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js index 9a41030a..3ad81962 100644 --- a/test/unit/cartodb/lzmaMiddleware.test.js +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -12,6 +12,7 @@ describe('lzma-middleware', function() { } }; testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { + const lzma = lzmaMiddleware(); var req = { headers: { host:'localhost' @@ -19,9 +20,13 @@ describe('lzma-middleware', function() { query: { api_key: 'test', lzma: data + }, + profiler: { + done: function () {} } }; - lzmaMiddleware(req, {}, function(err) { + + lzma(req, {}, function(err) { if ( err ) { return done(err); } diff --git a/test/unit/cartodb/ported/tile_stats.test.js b/test/unit/cartodb/ported/tile_stats.test.js deleted file mode 100644 index a2d30cd0..00000000 --- a/test/unit/cartodb/ported/tile_stats.test.js +++ /dev/null @@ -1,90 +0,0 @@ -require('../../../support/test_helper.js'); - -var assert = require('assert'); - -var LayergroupController = require('../../../../lib/cartodb/controllers/layergroup'); - -describe('tile stats', function() { - - beforeEach(function () { - this.statsClient = global.statsClient; - }); - - afterEach(function() { - global.statsClient = this.statsClient; - }); - - it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() { - var expectedCalls = 2, // it will call increment once for the general error - invalidFormat = 'png2', - invalidFormatRegexp = new RegExp('invalid'), - formatMatched = false; - mockStatsClientGetInstance({ - increment: function(label) { - formatMatched = formatMatched || !!label.match(invalidFormatRegexp); - expectedCalls--; - } - }); - - var layergroupController = new LayergroupController(); - - var reqMock = { - profiler: { toJSONString:function() {} }, - params: { - format: invalidFormat - } - }; - var resMock = { - status: function() { return this; }, - set: function() {}, - json: function() {}, - jsonp: function() {}, - send: function() {} - }; - - var next = function () {}; - layergroupController.finalizeGetTileOrGrid('Unsupported format png2', reqMock, resMock, null, null, next); - - assert.ok(formatMatched, 'Format was never matched in increment method'); - assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method'); - }); - - it('finalizeGetTileOrGrid calls statsClient when format is supported', function() { - var expectedCalls = 2, // general error + format error - validFormat = 'png', - validFormatRegexp = new RegExp(validFormat), - formatMatched = false; - mockStatsClientGetInstance({ - increment: function(label) { - formatMatched = formatMatched || !!label.match(validFormatRegexp); - expectedCalls--; - } - }); - var reqMock = { - profiler: { toJSONString:function() {} }, - params: { - format: validFormat - } - }; - var resMock = { - status: function() { return this; }, - set: function() {}, - json: function() {}, - jsonp: function() {}, - send: function() {} - }; - - var layergroupController = new LayergroupController(); - - var next = function () {}; - layergroupController.finalizeGetTileOrGrid('Another error happened', reqMock, resMock, null, null, next); - - assert.ok(formatMatched, 'Format was never matched in increment method'); - assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method'); - }); - - function mockStatsClientGetInstance(instance) { - global.statsClient = Object.assign(global.statsClient, instance); - } - -}); diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index b2864c1a..283c4fdc 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -10,7 +10,7 @@ var TemplateMaps = require('../../../lib/cartodb/backends/template_maps'); const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/context/clean-up-query-params'); const authorizeMiddleware = require('../../../lib/cartodb/middleware/context/authorize'); const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/context/db-conn-setup'); -const apikeyCredentialsMiddleware = require('../../../lib/cartodb/middleware/context/apikey-credentials'); +const credentialsMiddleware = require('../../../lib/cartodb/middleware/context/credentials'); const localsMiddleware = require('../../../lib/cartodb/middleware/context/locals'); var windshaft = require('windshaft'); @@ -24,7 +24,7 @@ describe('prepare-context', function() { let cleanUpQueryParams; let dbConnSetup; let authorize; - let setApikeyCredentials; + let setCredentials; before(function() { var redisPool = new RedisPool(global.environment.redis); @@ -37,7 +37,7 @@ describe('prepare-context', function() { cleanUpQueryParams = cleanUpQueryParamsMiddleware(); authorize = authorizeMiddleware(authApi); dbConnSetup = dbConnSetupMiddleware(pgConnection); - setApikeyCredentials = apikeyCredentialsMiddleware(); + setCredentials = credentialsMiddleware(); }); @@ -67,16 +67,17 @@ describe('prepare-context', function() { } it('res.locals are created', function(done) { + const locals = localsMiddleware(); let req = {}; let res = {}; - localsMiddleware(prepareRequest(req), prepareResponse(res), function(err) { + locals(prepareRequest(req), prepareResponse(res), function(err) { if ( err ) { done(err); return; } assert.ok(res.hasOwnProperty('locals'), 'response has locals'); done(); }); }); - + it('cleans up request', function(done){ var req = {headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}; var res = {}; @@ -108,18 +109,18 @@ describe('prepare-context', function() { }); it('sets also dbuser for authenticated requests', function(done){ - var req = { - headers: { - host: 'localhost' - }, + var req = { + headers: { + host: 'localhost' + }, query: { api_key: '1234' } }; - var res = { + var res = { set: function () {}, locals: { - api_key: '1234' + api_key: '1234' } }; @@ -171,7 +172,7 @@ describe('prepare-context', function() { } }; var res = {}; - + cleanUpQueryParams(prepareRequest(req), prepareResponse(res), function (err) { if ( err ) { return done(err); @@ -196,12 +197,12 @@ describe('prepare-context', function() { } }; var res = {}; - setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) { + setCredentials(prepareRequest(req), prepareResponse(res), function (err) { if (err) { return done(err); } var query = res.locals; - + assert.equal('1234', query.api_key); done(); }); @@ -217,7 +218,7 @@ describe('prepare-context', function() { } }; var res = {}; - setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) { + setCredentials(prepareRequest(req), prepareResponse(res), function (err) { if (err) { return done(err); } @@ -236,7 +237,7 @@ describe('prepare-context', function() { } }; var res = {}; - setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) { + setCredentials(prepareRequest(req), prepareResponse(res), function (err) { if (err) { return done(err); } diff --git a/yarn.lock b/yarn.lock index 0c7d698c..be24df2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11,11 +11,11 @@ node-pre-gyp "~0.6.30" protozero "1.5.1" -"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb1": - version "2.5.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/b0b5559f948e77b337bc9a9ae0bf6ec4249fba21" +"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb3": + version "2.5.1-cdb3" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/e61c7752c033595a273dcd1d4b267252b174bd28" dependencies: - "@carto/mapnik" "~3.6.2-carto.0" + "@carto/mapnik" "3.6.2-carto.2" "@mapbox/sphericalmercator" "~1.0.1" mapnik-pool "~0.1.3" @@ -23,7 +23,7 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -abaculus@cartodb/abaculus#2.0.3-cdb2: +"abaculus@github:cartodb/abaculus#2.0.3-cdb2": version "2.0.3-cdb2" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc" dependencies: @@ -249,7 +249,7 @@ camshaft@0.61.2: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -275,7 +275,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -2212,11 +2212,11 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb5: - version "0.6.18-cdb5" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cec846025e60837c60af193d600d972917ea8d35" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb7": + version "0.6.18-cdb7" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/488d2acd65c89cc5382d996eabe8dc1f5051ce0f" dependencies: - "@carto/mapnik" "~3.6.2-carto.0" + "@carto/mapnik" "3.6.2-carto.2" generic-pool "~2.4.0" mime "~1.6.0" sphericalmercator "~1.0.4" @@ -2373,12 +2373,12 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.5.3: - version "4.5.3" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.3.tgz#45e792af06b224f78f44b6eb3b0ecb9d90dcc943" +windshaft@4.5.4: + version "4.5.4" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.4.tgz#ce9b2f1bbc8ef749a26693e1832774b438d78cb9" dependencies: "@carto/mapnik" "3.6.2-carto.2" - "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb1 + "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb3 abaculus cartodb/abaculus#2.0.3-cdb2 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 @@ -2393,7 +2393,7 @@ windshaft@4.5.3: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb5 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb7 torque.js "~2.11.0" underscore "~1.6.0"