merge master

NEWS.md

@@ -2,9 +2,10 @@
 
 ## 5.4.0
 Released yyyy-mm-dd
- - Upgrades Windshaft to 4.5.3
+ - Upgrades Windshaft to 4.5.4 ([Mapnik top metrics](https://github.com/CartoDB/Windshaft/pull/597), [AttributesBackend allows multiple features if all the attributes are the same](https://github.com/CartoDB/Windshaft/pull/602))
  - Implemented middleware to authorize users via new Api Key system
  - Keep the old authorization system as fallback
+ - Aggregation widget: Remove NULL categories in 'count' aggregations too
 
 ## 5.3.1
 Released 2018-02-13

lib/cartodb/api/auth_api.js

@@ -62,7 +62,7 @@ function isValidApiKey(apikey) {
 //
 AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) {
     const apikeyToken = res.locals.api_key;
-    const apikeyUsername = res.locals.apikeyUsername;
+    const basicAuthUsername = res.locals.basicAuthUsername;
 
     if ( ! apikeyToken ) {
         return callback(null, false); // no api key, no authorization...
@@ -91,7 +91,7 @@ AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) {
             return callback(error);
         }
 
-        if (!usernameMatches(apikeyUsername, res.locals.user)) {
+        if (!usernameMatches(basicAuthUsername, res.locals.user)) {
             const error = new Error('Forbidden');
             error.type = 'auth';
             error.subtype = 'api-key-username-mismatch';
@@ -149,8 +149,8 @@ function isNameNotFoundError (err) {
     return err.message && -1 !== err.message.indexOf('name not found');
 }
 
-function usernameMatches (apikeyUsername, requestUsername) {
+function usernameMatches (basicAuthUsername, requestUsername) {
-    return !(apikeyUsername && (apikeyUsername !== requestUsername));
+    return !(basicAuthUsername && (basicAuthUsername !== requestUsername));
 }
 
 /**

lib/cartodb/controllers/analyses.js

@@ -12,7 +12,7 @@ AnalysesController.prototype.register = function (app) {
     app.get(
         `${app.base_url_mapconfig}/analyses/catalog`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         this.prepareContext,
         this.createPGClient(),
         this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),

lib/cartodb/controllers/layergroup.js

@@ -1,20 +1,22 @@
-var assert = require('assert');
+const cors = require('../middleware/cors');
-var step = require('step');
+const userMiddleware = require('../middleware/user');
-
+const allowQueryParams = require('../middleware/allow-query-params');
-var cors = require('../middleware/cors');
+const vectorError = require('../middleware/vector-error');
-var userMiddleware = require('../middleware/user');
+const DataviewBackend = require('../backends/dataview');
-var allowQueryParams = require('../middleware/allow-query-params');
+const AnalysisStatusBackend = require('../backends/analysis-status');
-var vectorError = require('../middleware/vector-error');
+const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
-
+const QueryTables = require('cartodb-query-tables');
-var DataviewBackend = require('../backends/dataview');
+const SUPPORTED_FORMATS = {
-var AnalysisStatusBackend = require('../backends/analysis-status');
+    grid_json: true,
-
+    json_torque: true,
-var MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
+    torque_json: true,
-
+    png: true,
-var QueryTables = require('cartodb-query-tables');
+    png32: true,
+    mvt: true
+};
 
 /**
- * @param {AuthApi} authApi
+ * @param {prepareContext} prepareContext
  * @param {PgConnection} pgConnection
  * @param {MapStore} mapStore
  * @param {TileBackend} tileBackend
@@ -46,64 +48,119 @@ function LayergroupController(prepareContext, pgConnection, mapStore, tileBacken
 module.exports = LayergroupController;
 
 LayergroupController.prototype.register = function(app) {
+    const { base_url_mapconfig: basePath } = app;
+
     app.get(
-        app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format',
+        `${basePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         this.prepareContext,
-        this.tile.bind(this),
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        getTile(this.tileBackend, 'map_tile'),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        incrementSuccessMetrics(global.statsClient),
+        sendResponse(),
+        incrementErrorMetrics(global.statsClient),
+        tileError(),
         vectorError()
     );
 
     app.get(
-        app.base_url_mapconfig + '/:token/:z/:x/:y.:format',
+        `${basePath}/:token/:z/:x/:y.:format`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         this.prepareContext,
-        this.tile.bind(this),
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        getTile(this.tileBackend, 'map_tile'),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        incrementSuccessMetrics(global.statsClient),
+        sendResponse(),
+        incrementErrorMetrics(global.statsClient),
+        tileError(),
         vectorError()
     );
 
     app.get(
-        app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)',
+        `${basePath}/:token/:layer/:z/:x/:y.(:format)`,
+        distinguishLayergroupFromStaticRoute(),
         cors(),
-        userMiddleware,
+        userMiddleware(),
-        validateLayerRouteMiddleware,
         this.prepareContext,
-        this.layer.bind(this),
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        getTile(this.tileBackend, 'maplayer_tile'),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        incrementSuccessMetrics(global.statsClient),
+        sendResponse(),
+        incrementErrorMetrics(global.statsClient),
+        tileError(),
         vectorError()
     );
 
     app.get(
-        app.base_url_mapconfig + '/:token/:layer/attributes/:fid',
+        `${basePath}/:token/:layer/attributes/:fid`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         this.prepareContext,
-        this.attributes.bind(this)
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        getFeatureAttributes(this.attributesBackend),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        sendResponse()
     );
 
+    const forcedFormat = 'png';
+
     app.get(
-        app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
+        `${basePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(['layer']),
         this.prepareContext,
-        this.center.bind(this)
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
+        getPreviewImageByCenter(this.previewBackend),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        sendResponse()
     );
 
     app.get(
-        app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
+        `${basePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(['layer']),
         this.prepareContext,
-        this.bbox.bind(this)
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
+        getPreviewImageByBoundingBox(this.previewBackend),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        sendResponse()
     );
 
     // Undocumented/non-supported API endpoint methods.
     // Use at your own peril.
 
-    var allowedDataviewQueryParams = [
+    const allowedDataviewQueryParams = [
         'filters', // json
         'own_filter', // 0, 1
         'no_filters', // 0, 1
@@ -119,395 +176,459 @@ LayergroupController.prototype.register = function(app) {
     ];
 
     app.get(
-        app.base_url_mapconfig + '/:token/dataview/:dataviewName',
+        `${basePath}/:token/dataview/:dataviewName`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(allowedDataviewQueryParams),
         this.prepareContext,
-        this.dataview.bind(this)
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        getDataview(this.dataviewBackend),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        sendResponse()
     );
 
     app.get(
-        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
+        `${basePath}/:token/:layer/widget/:dataviewName`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(allowedDataviewQueryParams),
         this.prepareContext,
-        this.dataview.bind(this)
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        getDataview(this.dataviewBackend),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        sendResponse()
     );
 
     app.get(
-        app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
+        `${basePath}/:token/dataview/:dataviewName/search`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(allowedDataviewQueryParams),
         this.prepareContext,
-        this.dataviewSearch.bind(this)
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        dataviewSearch(this.dataviewBackend),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        sendResponse()
     );
 
     app.get(
-        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
+        `${basePath}/:token/:layer/widget/:dataviewName/search`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(allowedDataviewQueryParams),
         this.prepareContext,
-        this.dataviewSearch.bind(this)
+        createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
+        dataviewSearch(this.dataviewBackend),
+        setCacheControlHeader(),
+        setLastModifiedHeader(),
+        getAffectedTables(this.layergroupAffectedTables, this.pgConnection),
+        setCacheChannelHeader(),
+        setSurrogateKeyHeader(this.surrogateKeysCache),
+        sendResponse()
     );
 
     app.get(
-        app.base_url_mapconfig + '/:token/analysis/node/:nodeId',
+        `${basePath}/:token/analysis/node/:nodeId`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
         this.prepareContext,
-        this.analysisNodeStatus.bind(this)
+        analysisNodeStatus(this.analysisStatusBackend),
+        sendResponse()
     );
 };
 
-LayergroupController.prototype.analysisNodeStatus = function(req, res, next) {
+function distinguishLayergroupFromStaticRoute () {
-    var self = this;
+    return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) {
+        if (req.params.token === 'static') {
+            return next('route');
+        }
 
-    step(
+        next();
-        function retrieveNodeStatus() {
+    };
-            self.analysisStatusBackend.getNodeStatus(res.locals, this);
+}
-        },
+
-        function finish(err, nodeStatus, stats) {
+function analysisNodeStatus (analysisStatusBackend) {
-            req.profiler.add(stats || {});
+    return function analysisNodeStatusMiddleware(req, res, next) {
+        analysisStatusBackend.getNodeStatus(res.locals, (err, nodeStatus, stats = {}) => {
+            req.profiler.add(stats);
 
             if (err) {
                 err.label = 'GET NODE STATUS';
-                next(err);
+                return next(err);
-            } else {
-                self.sendResponse(req, res, nodeStatus, 200, {
-                    'Cache-Control': 'public,max-age=5',
-                    'Last-Modified': new Date().toUTCString()
-                });
             }
+
+            res.set({
+                'Cache-Control': 'public,max-age=5',
+                'Last-Modified': new Date().toUTCString()
+            });
+
+            res.body = nodeStatus;
+
+            next();
+        });
+    };
+}
+
+function getRequestParams(locals) {
+    const params = Object.assign({}, locals);
+
+    delete params.mapConfigProvider;
+    delete params.allowedQueryParams;
+
+    return params;
+}
+
+function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) {
+    return function createMapStoreMapConfigProviderMiddleware (req, res, next) {
+        const { user } = res.locals;
+
+        const params = getRequestParams(res.locals);
+
+        if (forcedFormat) {
+            params.format = forcedFormat;
+            params.layer = params.layer || 'all';
         }
-    );
-};
 
-LayergroupController.prototype.dataview = function(req, res, next) {
+        const mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params);
-    var self = this;
 
-    step(
+        mapConfigProvider.getMapConfig((err, mapconfig) => {
-        function retrieveDataview() {
+            if (err) {
-            var mapConfigProvider = new MapStoreMapConfigProvider(
+                return next(err);
-                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
+            }
-            );
+
-            self.dataviewBackend.getDataview(
+            res.locals.mapConfigProvider = mapConfigProvider;
-                mapConfigProvider,
+            res.locals.mapconfig = mapconfig;
-                res.locals.user,
+
-                res.locals,
+            next();
-                this
+        });
-            );
+    };
-        },
+}
-        function finish(err, dataview, stats) {
+
-            req.profiler.add(stats || {});
+function getDataview (dataviewBackend) {
+    return function getDataviewMiddleware (req, res, next) {
+        const { user, mapConfigProvider } = res.locals;
+        const params = getRequestParams(res.locals);
+
+        dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => {
+            req.profiler.add(stats);
 
             if (err) {
                 err.label = 'GET DATAVIEW';
-                next(err);
+                return next(err);
-            } else {
-                self.sendResponse(req, res, dataview, 200);
             }
-        }
-    );
-};
 
-LayergroupController.prototype.dataviewSearch = function(req, res, next) {
+            res.body = dataview;
-    var self = this;
 
-    step(
+            next();
-        function searchDataview() {
+        });
-            var mapConfigProvider = new MapStoreMapConfigProvider(
+    };
-                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
+}
-            );
+
-            self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this);
+function dataviewSearch (dataviewBackend) {
-        },
+    return function dataviewSearchMiddleware (req, res, next) {
-        function finish(err, searchResult, stats) {
+        const { user, dataviewName, mapConfigProvider } = res.locals;
-            req.profiler.add(stats || {});
+        const params = getRequestParams(res.locals);
+
+        dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => {
+            req.profiler.add(stats);
 
             if (err) {
                 err.label = 'GET DATAVIEW SEARCH';
-                next(err);
+                return next(err);
-            } else {
-                self.sendResponse(req, res, searchResult, 200);
             }
-        }
-    );
 
-};
+            res.body = searchResult;
 
-LayergroupController.prototype.attributes = function(req, res, next) {
+            next();
-    var self = this;
+        });
+    };
+}
 
-    req.profiler.start('windshaft.maplayer_attribute');
+function getFeatureAttributes (attributesBackend) {
+    return function getFeatureAttributesMiddleware (req, res, next) {
+        req.profiler.start('windshaft.maplayer_attribute');
 
-    step(
+        const { mapConfigProvider } = res.locals;
-        function retrieveFeatureAttributes() {
+        const params = getRequestParams(res.locals);
-            var mapConfigProvider = new MapStoreMapConfigProvider(
+
-                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
+        attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => {
-            );
+            req.profiler.add(stats);
-            self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this);
-        },
-        function finish(err, tile, stats) {
-            req.profiler.add(stats || {});
 
             if (err) {
                 err.label = 'GET ATTRIBUTES';
-                next(err);
+                return next(err);
-            } else {
-                self.sendResponse(req, res, tile, 200);
             }
-        }
-    );
 
-};
+            res.body = tile;
 
-// Gets a tile for a given token and set of tile ZXY coords. (OSM style)
+            next();
-LayergroupController.prototype.tile = function(req, res, next) {
+        });
-    req.profiler.start('windshaft.map_tile');
+    };
-    this.tileOrLayer(req, res, next);
-};
-
-// Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
-LayergroupController.prototype.layer = function(req, res, next) {
-    req.profiler.start('windshaft.maplayer_tile');
-    this.tileOrLayer(req, res, next);
-};
-
-LayergroupController.prototype.tileOrLayer = function (req, res, next) {
-    var self = this;
-
-    step(
-        function mapController$getTileOrGrid() {
-            self.tileBackend.getTile(
-                new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
-                res.locals, this
-            );
-        },
-        function mapController$finalize(err, tile, headers, stats) {
-            req.profiler.add(stats);
-            self.finalizeGetTileOrGrid(err, req, res, tile, headers, next);
-        }
-    );
-};
-
-function getStatusCode(tile, format){
-    return tile.length===0 && format==='mvt'? 204:200;
 }
 
-// This function is meant for being called as the very last
+function getStatusCode(tile, format){
-// step by all endpoints serving tiles or grids
+    return tile.length === 0 && format === 'mvt'? 204 : 200;
-LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) {
+}
-    var supportedFormats = {
-        grid_json: true,
-        json_torque: true,
-        torque_json: true,
-        png: true,
-        png32: true,
-        mvt: true
-    };
 
-    var formatStat = 'invalid';
+function parseFormat (format = '') {
-    if (req.params.format) {
+    const prettyFormat = format.replace('.', '_');
-        var format = req.params.format.replace('.', '_');
+    return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid';
-        if (supportedFormats[format]) {
+}
-            formatStat = format;
-        }
-    }
 
-    if (err) {
+function getTile (tileBackend, profileLabel = 'tile') {
-        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    return function getTileMiddleware (req, res, next) {
-        var errMsg = err.message ? ( '' + err.message ) : ( '' + err );
+        req.profiler.start(`windshaft.${profileLabel}`);
 
-        // Rewrite mapnik parsing errors to start with layer number
+        const { mapConfigProvider } = res.locals;
-        var matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
+        const params = getRequestParams(res.locals);
-        if (matches) {
-            errMsg = 'style'+matches[2]+': ' + matches[1];
-        }
-        err.message = errMsg;
 
-        err.label = 'TILE RENDER';
+        tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
-        next(err);
+            req.profiler.add(stats);
-
-        global.statsClient.increment('windshaft.tiles.error');
-        global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
-    } else {
-        this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers);
-        global.statsClient.increment('windshaft.tiles.success');
-        global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
-    }
-};
-
-LayergroupController.prototype.bbox = function(req, res, next) {
-    this.staticMap(req, res, +req.params.width, +req.params.height, {
-        west: +req.params.west,
-        north: +req.params.north,
-        east: +req.params.east,
-        south: +req.params.south
-    }, null, next);
-};
-
-LayergroupController.prototype.center = function(req, res, next) {
-    this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
-        lng: +req.params.lng,
-        lat: +req.params.lat
-    }, next);
-};
-
-LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) {
-    var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    // We force always the tile to be generated using PNG because
-    // is the only format we support by now
-    res.locals.format = 'png';
-    res.locals.layer = res.locals.layer || 'all';
-
-    var self = this;
-
-    step(
-        function getImage() {
-            if (center) {
-                self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
-                    format, width, height, zoom, center, this);
-            } else {
-                self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
-                    format, width, height, zoom /* bounds */, this);
-            }
-        },
-        function handleImage(err, image, headers, stats) {
-            req.profiler.done('render-' + format);
-            req.profiler.add(stats || {});
 
             if (err) {
-                err.label = 'STATIC_MAP';
+                return next(err);
-                next(err);
-            } else {
-                res.set('Content-Type', headers['Content-Type'] || 'image/' + format);
-                self.sendResponse(req, res, image, 200);
-            }
-        }
-    );
-};
-
-LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
-    var self = this;
-
-    req.profiler.done('res');
-
-    res.set('Cache-Control', 'public,max-age=31536000');
-
-    // Set Last-Modified header
-    var lastUpdated;
-    if (res.locals.cache_buster) {
-        // Assuming cache_buster is a timestamp
-        lastUpdated = new Date(parseInt(res.locals.cache_buster));
-    } else {
-        lastUpdated = new Date();
-    }
-    res.set('Last-Modified', lastUpdated.toUTCString());
-
-    var dbName = res.locals.dbname;
-    step(
-        function getAffectedTables() {
-            self.getAffectedTables(res.locals.user, dbName, res.locals.token, this);
-        },
-        function sendResponse(err, affectedTables) {
-            req.profiler.done('affectedTables');
-            if (err) {
-                global.logger.warn('ERROR generating cache channel: ' + err);
-            }
-            if (!!affectedTables) {
-                res.set('X-Cache-Channel', affectedTables.getCacheChannel());
-                self.surrogateKeysCache.tag(res, affectedTables);
             }
 
             if (headers) {
                 res.set(headers);
             }
 
-            res.status(status);
+            const formatStat = parseFormat(req.params.format);
 
-            if (!Buffer.isBuffer(body) && typeof body === 'object') {
+            res.statusCode = getStatusCode(tile, formatStat);
-                if (req.query && req.query.callback) {
+            res.body = tile;
-                    res.jsonp(body);
-                } else {
-                    res.json(body);
-                }
-            } else {
-                res.send(body);
-            }
-        }
-    );
-};
 
-LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
+            next();
-
+        });
-    if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) {
+    };
-        return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId));
+}
-    }
+
-
+function getPreviewImageByCenter (previewBackend) {
-    var self = this;
+    return function getPreviewImageByCenterMiddleware (req, res, next) {
-    step(
+        const width = +req.params.width;
-        function extractSQL() {
+        const height = +req.params.height;
-            step(
+        const zoom = +req.params.z;
-                function loadFromStore() {
+        const center = {
-                    self.mapStore.load(layergroupId, this);
+            lng: +req.params.lng,
-                },
+            lat: +req.params.lat
-                function getSQL(err, mapConfig) {
+        };
-                    assert.ifError(err);
+
-
+        const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-                    var queries = [];
+        const { mapConfigProvider: provider } = res.locals;
-                    mapConfig.getLayers().forEach(function(layer) {
+
-                        queries.push(layer.options.sql);
+        previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => {
-                        if (layer.options.affected_tables) {
+            req.profiler.done(`render-${format}`);
-                            layer.options.affected_tables.map(function(table) {
+            req.profiler.add(stats);
-                                queries.push('SELECT * FROM ' + table + ' LIMIT 0');
+
-                            });
+            if (err) {
-                        }
+                err.label = 'STATIC_MAP';
-                    });
+                return next(err);
-
+            }
-                    return queries.length ? queries.join(';') : null;
+
-                },
+            if (headers) {
-                this
+                res.set(headers);
-            );
+            }
-        },
+
-        function findAffectedTables(err, sql) {
+            res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
-            assert.ifError(err);
+
-
+            res.body = image;
-            if ( ! sql ) {
+
-                throw new Error("this request doesn't need an X-Cache-Channel generated");
+            next();
-            }
+        });
-
+    };
-            step(
+}
-                function getConnection() {
+
-                    self.pgConnection.getConnection(user, this);
+function getPreviewImageByBoundingBox (previewBackend) {
-                },
+    return function getPreviewImageByBoundingBoxMiddleware (req, res, next) {
-                function getAffectedTables(err, connection) {
+        const width = +req.params.width;
-                    assert.ifError(err);
+        const height = +req.params.height;
-
+        const bounds = {
-                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+            west: +req.params.west,
-                },
+            north: +req.params.north,
-                this
+            east: +req.params.east,
-            );
+            south: +req.params.south
-        },
+        };
-        function buildCacheChannel(err, tables) {
+        const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-            assert.ifError(err);
+        const { mapConfigProvider: provider } = res.locals;
-            self.layergroupAffectedTables.set(dbName, layergroupId, tables);
+
-
+        previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => {
-            return tables;
+            req.profiler.done(`render-${format}`);
-        },
+            req.profiler.add(stats);
-        callback
+
-    );
+            if (err) {
-};
+                err.label = 'STATIC_MAP';
-
+                return next(err);
-
+            }
-function validateLayerRouteMiddleware(req, res, next) {
+
-    if (req.params.token === 'static') {
+            if (headers) {
-        return next('route');
+                res.set(headers);
-    }
+            }
-
+
-    next();
+            res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
+
+            res.body = image;
+
+            next();
+        });
+    };
+}
+
+function setLastModifiedHeader () {
+    return function setLastModifiedHeaderMiddleware (req, res, next) {
+        let { cache_buster: cacheBuster } = res.locals;
+
+        cacheBuster = parseInt(cacheBuster, 10);
+
+        const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date();
+
+        res.set('Last-Modified', lastUpdated.toUTCString());
+
+        next();
+    };
+}
+
+function setCacheControlHeader () {
+    return function setCacheControlHeaderMiddleware (req, res, next) {
+        res.set('Cache-Control', 'public,max-age=31536000');
+
+        next();
+    };
+}
+
+function getAffectedTables (layergroupAffectedTables, pgConnection) {
+    return function getAffectedTablesMiddleware (req, res, next) {
+        const { user, dbname, token, mapconfig } = res.locals;
+
+        if (layergroupAffectedTables.hasAffectedTables(dbname, token)) {
+            res.locals.affectedTables = layergroupAffectedTables.get(dbname, token);
+            return next();
+        }
+
+        pgConnection.getConnection(user, (err, connection) => {
+            if (err) {
+                global.logger.warn('ERROR generating cache channel:', err);
+                return next();
+            }
+
+            const sql = [];
+            mapconfig.getLayers().forEach(function(layer) {
+                sql.push(layer.options.sql);
+                if (layer.options.affected_tables) {
+                    layer.options.affected_tables.map(function(table) {
+                        sql.push(`SELECT * FROM ${table} LIMIT 0`);
+                    });
+                }
+            });
+
+            QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => {
+                req.profiler.done('getAffectedTablesFromQuery');
+                if (err) {
+                    global.logger.warn('ERROR generating cache channel: ', err);
+                    return next();
+                }
+
+                // feed affected tables cache so it can be reused from, for instance, map controller
+                layergroupAffectedTables.set(dbname, token, affectedTables);
+
+                res.locals.affectedTables = affectedTables;
+
+                next();
+            });
+        });
+    };
+}
+
+function setCacheChannelHeader () {
+    return function setCacheChannelHeaderMiddleware (req, res, next) {
+        const { affectedTables } = res.locals;
+
+        if (affectedTables) {
+            res.set('X-Cache-Channel', affectedTables.getCacheChannel());
+        }
+
+        next();
+    };
+}
+
+function setSurrogateKeyHeader (surrogateKeysCache) {
+    return function setSurrogateKeyHeaderMiddleware (req, res, next) {
+        const { affectedTables } = res.locals;
+
+        if (affectedTables) {
+            surrogateKeysCache.tag(res, affectedTables);
+        }
+
+        next();
+    };
+}
+
+function incrementSuccessMetrics (statsClient) {
+    return function incrementSuccessMetricsMiddleware (req, res, next) {
+        const formatStat = parseFormat(req.params.format);
+
+        statsClient.increment('windshaft.tiles.success');
+        statsClient.increment(`windshaft.tiles.${formatStat}.success`);
+
+        next();
+    };
+}
+
+function sendResponse () {
+    return function sendResponseMiddleware (req, res) {
+        req.profiler.done('res');
+
+        res.status(res.statusCode || 200);
+
+        if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') {
+            if (req.query && req.query.callback) {
+                res.jsonp(res.body);
+            } else {
+                res.json(res.body);
+            }
+        } else {
+            res.send(res.body);
+        }
+    };
+}
+
+function incrementErrorMetrics (statsClient) {
+    return function incrementErrorMetricsMiddleware (err, req, res, next) {
+        const formatStat = parseFormat(req.params.format);
+
+        statsClient.increment('windshaft.tiles.error');
+        statsClient.increment(`windshaft.tiles.${formatStat}.error`);
+
+        next(err);
+    };
+}
+
+function tileError () {
+    return function tileErrorMiddleware (err, req, res, next) {
+        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+        let errMsg = err.message ? ( '' + err.message ) : ( '' + err );
+
+        // Rewrite mapnik parsing errors to start with layer number
+        const matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
+
+        if (matches) {
+            errMsg = `style${matches[2]}: ${matches[1]}`;
+        }
+
+        err.message = errMsg;
+        err.label = 'TILE RENDER';
+
+        next(err);
+    };
 }

lib/cartodb/controllers/map.js

@@ -69,7 +69,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal
 
     return [
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(['aggregation']),
         this.prepareContext,
         this.initProfiler(isTemplateInstantiation),

lib/cartodb/controllers/named_maps.js

@@ -48,7 +48,7 @@ NamedMapsController.prototype.register = function(app) {
     app.get(
         app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)',
         cors(),
-        userMiddleware,
+        userMiddleware(),
         this.prepareContext,
         this.getNamedMapProvider(tileOptions),
         this.getAffectedTables(),
@@ -70,7 +70,7 @@ NamedMapsController.prototype.register = function(app) {
     app.get(
         app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
         cors(),
-        userMiddleware,
+        userMiddleware(),
         allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
         this.prepareContext,
         this.getNamedMapProvider(staticOptions),

lib/cartodb/controllers/named_maps_admin.js

@@ -2,12 +2,7 @@ const { templateName } = require('../backends/template_maps');
 const cors = require('../middleware/cors');
 const userMiddleware = require('../middleware/user');
 const localsMiddleware = require('../middleware/context/locals');
-const apikeyCredentialsMiddleware = require('../middleware/context/apikey-credentials');
+const credentialsMiddleware = require('../middleware/context/credentials');
-
-const apikeyMiddleware = [
-    localsMiddleware,
-    apikeyCredentialsMiddleware(),
-];
 
 /**
  * @param {AuthApi} authApi
@@ -28,8 +23,9 @@ NamedMapsAdminController.prototype.register = function (app) {
     app.post(
         `${base_url_templated}/`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
-        apikeyMiddleware,
+        localsMiddleware(),
+        credentialsMiddleware(),
         this.checkContentType('POST', 'POST TEMPLATE'),
         this.authorizedByAPIKey('create', 'POST TEMPLATE'),
         this.create()
@@ -38,8 +34,9 @@ NamedMapsAdminController.prototype.register = function (app) {
     app.put(
         `${base_url_templated}/:template_id`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
-        apikeyMiddleware,
+        localsMiddleware(),
+        credentialsMiddleware(),
         this.checkContentType('PUT', 'PUT TEMPLATE'),
         this.authorizedByAPIKey('update', 'PUT TEMPLATE'),
         this.update()
@@ -48,8 +45,9 @@ NamedMapsAdminController.prototype.register = function (app) {
     app.get(
         `${base_url_templated}/:template_id`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
-        apikeyMiddleware,
+        localsMiddleware(),
+        credentialsMiddleware(),
         this.authorizedByAPIKey('get', 'GET TEMPLATE'),
         this.retrieve()
     );
@@ -57,8 +55,9 @@ NamedMapsAdminController.prototype.register = function (app) {
     app.delete(
         `${base_url_templated}/:template_id`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
-        apikeyMiddleware,
+        localsMiddleware(),
+        credentialsMiddleware(),
         this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'),
         this.destroy()
     );
@@ -66,8 +65,9 @@ NamedMapsAdminController.prototype.register = function (app) {
     app.get(
         `${base_url_templated}/`,
         cors(),
-        userMiddleware,
+        userMiddleware(),
-        apikeyMiddleware,
+        localsMiddleware(),
+        credentialsMiddleware(),
         this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'),
         this.list()
     );

lib/cartodb/middleware/allow-query-params.js

@@ -1,8 +1,9 @@
-module.exports = function allowQueryParams(params) {
+module.exports = function allowQueryParams (params) {
     if (!Array.isArray(params)) {
         throw new Error('allowQueryParams must receive an Array of params');
     }
-    return function allowQueryParamsMiddleware(req, res, next) {
+
+    return function allowQueryParamsMiddleware (req, res, next) {
         res.locals.allowedQueryParams = params;
         next();
     };

lib/cartodb/middleware/context/authorize.js

@@ -1,9 +1,8 @@
-module.exports = function authorizeMiddleware (authApi) {
+module.exports = function authorize (authApi) {
-    return function (req, res, next) {
+    return function authorizeMiddleware (req, res, next) {
-        req.profiler.done('req2params.setup');
-
         authApi.authorize(req, res, (err, authorized) => {
             req.profiler.done('authorize');
+
             if (err) {
                 return next(err);
             }

lib/cartodb/middleware/context/credentials.js

@@ -1,19 +1,17 @@
-'use strict';
+const basicAuth = require('basic-auth');
 
-module.exports = function apikeyToken () {
+module.exports = function credentials () {
-    return function apikeyTokenMiddleware(req, res, next) {
+    return function credentialsMiddleware(req, res, next) {
         const apikeyCredentials = getApikeyCredentialsFromRequest(req);
+
         res.locals.api_key = apikeyCredentials.token;
-        res.locals.apikeyUsername = apikeyCredentials.username;
+        res.locals.basicAuthUsername = apikeyCredentials.username;
-        res.set('vary', 'Authorization'); //Honor Authorization header when caching. 
+        res.set('vary', 'Authorization'); //Honor Authorization header when caching.
+
         return next();
     };
 };
 
-//--------------------------------------------------------------------------------
-
-const basicAuth = require('basic-auth');
-
 function getApikeyCredentialsFromRequest(req) {
     let apikeyCredentials = {
         token: null,

lib/cartodb/middleware/context/db-conn-setup.js

@@ -1,14 +1,17 @@
 const _ = require('underscore');
 
-module.exports = function dbConnSetupMiddleware(pgConnection) {
+module.exports = function dbConnSetup (pgConnection) {
-    return function dbConnSetup(req, res, next) {
+    return function dbConnSetupMiddleware (req, res, next) {
-        const user = res.locals.user;
+        const { user } = res.locals;
+
         pgConnection.setDBConn(user, res.locals, (err) => {
+            req.profiler.done('dbConnSetup');
+
             if (err) {
                 if (err.message && -1 !== err.message.indexOf('name not found')) {
                     err.http_status = 404;
                 }
-                req.profiler.done('req2params');
+
                 return next(err);
             }
 
@@ -18,12 +21,10 @@ module.exports = function dbConnSetupMiddleware(pgConnection) {
                 dbhost: global.environment.postgres.host,
                 dbport: global.environment.postgres.port
             });
-            
+
             res.set('X-Served-By-DB-Host', res.locals.dbhost);
 
-            req.profiler.done('req2params');
+            next();
-            
-            next(null);
         });
     };
 };

lib/cartodb/middleware/context/index.js

@@ -1,16 +1,16 @@
 const locals = require('./locals');
 const cleanUpQueryParams = require('./clean-up-query-params');
 const layergroupToken = require('./layergroup-token');
-const apikeyCredentials = require('./apikey-credentials');
+const credentials = require('./credentials');
 const authorize = require('./authorize');
 const dbConnSetup = require('./db-conn-setup');
 
 module.exports = function prepareContextMiddleware(authApi, pgConnection) {
     return [
-        locals,
+        locals(),
         cleanUpQueryParams(),
-        layergroupToken,
+        layergroupToken(),
-        apikeyCredentials(),
+        credentials(),
         authorize(authApi),
         dbConnSetup(pgConnection)
     ];

lib/cartodb/middleware/context/layergroup-token.js

@@ -1,32 +1,33 @@
-var LayergroupToken = require('../../models/layergroup-token');
+const LayergroupToken = require('../../models/layergroup-token');
-
+const authErrorMessageTemplate = function (signer, user) {
-module.exports = function layergroupTokenMiddleware(req, res, next) {
+    return `Cannot use map signature of user "${signer}" on db of user "${user}"`;
-    if (!res.locals.token) {
+};
-        return next();
+
-    }
+module.exports = function layergroupToken () {
-
+    return function layergroupTokenMiddleware (req, res, next) {
-    var user = res.locals.user;
+        if (!res.locals.token) {
-
+            return next();
-    var layergroupToken = LayergroupToken.parse(res.locals.token);
+        }
-    res.locals.token = layergroupToken.token;
+
-    res.locals.cache_buster = layergroupToken.cacheBuster;
+        const user = res.locals.user;
-
+
-    if (layergroupToken.signer) {
+        const layergroupToken = LayergroupToken.parse(res.locals.token);
-        res.locals.signer = layergroupToken.signer;
+
-        if (!res.locals.signer) {
+        res.locals.token = layergroupToken.token;
-            res.locals.signer = user;
+        res.locals.cache_buster = layergroupToken.cacheBuster;
-        } else if (res.locals.signer !== user) {
+
-            var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`);
+        if (layergroupToken.signer) {
-            err.type = 'auth';
+            res.locals.signer = layergroupToken.signer;
-            err.http_status = 403;
+
-            if (req.query && req.query.callback) {
+            if (res.locals.signer !== user) {
-                err.http_status = 200;
+                const err = new Error(authErrorMessageTemplate(res.locals.signer, user));
-            }
+                err.type = 'auth';
-            
+                err.http_status = (req.query && req.query.callback) ? 200: 403;
-            req.profiler.done('req2params');
+
-            return next(err);
+                return next(err);
-        }
+            }
-    }
+        }
-
+
-    return next();
+        return next();
+    };
 };

lib/cartodb/middleware/context/locals.js

@@ -1,6 +1,7 @@
-module.exports = function localsMiddleware(req, res, next) {
+module.exports = function locals () {
-    // save req.params in res.locals
+    return function localsMiddleware (req, res, next) {
-    res.locals = Object.assign(req.params || {}, res.locals);
+        res.locals = Object.assign(req.params || {}, res.locals);
 
-    next();
+        next();
+    };
 };

lib/cartodb/middleware/cors.js

@@ -1,11 +1,14 @@
-module.exports = function cors(extraHeaders) {
+module.exports = function cors (extraHeaders) {
-    return function(req, res, next) {
+    return function corsMiddleware (req, res, next) {
-        var baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token";
+        let baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token";
+
         if(extraHeaders) {
             baseHeaders += ", " + extraHeaders;
         }
+
         res.set("Access-Control-Allow-Origin", "*");
         res.set("Access-Control-Allow-Headers", baseHeaders);
+
         next();
     };
 };

lib/cartodb/middleware/lzma.js

@@ -1,30 +1,33 @@
-'use strict';
-
 const LZMA = require('lzma').LZMA;
 
-const lzmaWorker = new LZMA();
+module.exports = function lzma () {
+    const lzmaWorker = new LZMA();
 
-module.exports = function lzmaMiddleware(req, res, next) {
+    return function lzmaMiddleware (req, res, next) {
-    if (!req.query.hasOwnProperty('lzma')) {
+        if (!req.query.hasOwnProperty('lzma')) {
-        return next();
+            return next();
-    }
-
-    // Decode (from base64)
-    var lzma = new Buffer(req.query.lzma, 'base64')
-        .toString('binary')
-        .split('')
-        .map(function(c) {
-            return c.charCodeAt(0) - 128;
-        });
-
-    // Decompress
-    lzmaWorker.decompress(lzma, function(result) {
-        try {
-            delete req.query.lzma;
-            Object.assign(req.query, JSON.parse(result));
-            next();
-        } catch (err) {
-            next(new Error('Error parsing lzma as JSON: ' + err));
         }
-    });
+
+        // Decode (from base64)
+        var lzma = new Buffer(req.query.lzma, 'base64')
+            .toString('binary')
+            .split('')
+            .map(function(c) {
+                return c.charCodeAt(0) - 128;
+            });
+
+        // Decompress
+        lzmaWorker.decompress(lzma, function(result) {
+            try {
+                delete req.query.lzma;
+                Object.assign(req.query, JSON.parse(result));
+
+                req.profiler.done('lzma');
+
+                next();
+            } catch (err) {
+                next(new Error('Error parsing lzma as JSON: ' + err));
+            }
+        });
+    };
 };

lib/cartodb/middleware/stats.js

@@ -2,10 +2,10 @@ const Profiler = require('../stats/profiler_proxy');
 const debug = require('debug')('windshaft:cartodb:stats');
 const onHeaders = require('on-headers');
 
-module.exports = function statsMiddleware(options) {
+module.exports = function stats (options) {
     const { enabled = true, statsClient } = options;
 
-    return function stats(req, res, next) {
+    return function statsMiddleware (req, res, next) {
         req.profiler = new Profiler({
             statsd_client: statsClient,
             profile: enabled

lib/cartodb/middleware/user.js

@@ -1,8 +1,11 @@
-var CdbRequest = require('../models/cdb_request');
+const CdbRequest = require('../models/cdb_request');
-var cdbRequest = new CdbRequest();
 
-module.exports = function userMiddleware(req, res, next) {
+module.exports = function user () {
-    res.locals.user = cdbRequest.userByReq(req);
+    const cdbRequest = new CdbRequest();
 
-    next();
+    return function userMiddleware(req, res, next) {
+        res.locals.user = cdbRequest.userByReq(req);
+
+        next();
+    };
 };

lib/cartodb/middleware/vector-error.js

@@ -1,5 +1,4 @@
 const fs = require('fs');
-
 const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt');
 
 module.exports = function vectorError() {

lib/cartodb/models/dataview/aggregation.js

@@ -42,7 +42,7 @@ const rankedCategoriesQueryTpl = ctx => `
             ${ctx.aggregationFn} AS value,
             row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
         FROM (${filteredQueryTpl(ctx)}) filtered_source
-        ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
+        WHERE ${ctx.aggregation === "count" ? `${ctx.column}` : `${ctx.aggregationColumn}`} IS NOT NULL
         GROUP BY ${ctx.column}
         ORDER BY 2 DESC
     )
@@ -279,7 +279,7 @@ module.exports = class Aggregation extends BaseDataview {
             max_val = 0,
             categories_count = 0
          } = result.rows[0] || {};
-        
+
         return {
             aggregation: this.aggregation,
             count: count,
@@ -290,10 +290,10 @@ module.exports = class Aggregation extends BaseDataview {
             max: max_val,
             categoriesCount: categories_count,
             categories: result.rows.map(({ category, value, agg }) => {
-                return { 
+                return {
                     category: agg ? 'Other' : category,
-                    value, 
+                    value,
-                    agg 
+                    agg
                 };
             })
         };

lib/cartodb/server.js

@@ -377,7 +377,7 @@ function bootstrap(opts) {
         statsClient: global.statsClient
     }));
 
-    app.use(lzmaMiddleware);
+    app.use(lzmaMiddleware());
 
     // temporary measure until we upgrade to newer version expressjs so we can check err.status
     app.use(function(err, req, res, next) {

package.json

@@ -48,7 +48,7 @@
     "step-profiler": "~0.3.0",
     "turbo-carto": "0.20.2",
     "underscore": "~1.6.0",
-    "windshaft": "4.5.3",
+    "windshaft": "4.5.4",
     "yargs": "~5.0.0"
   },
   "devDependencies": {

test/acceptance/dataviews/aggregation.js

@@ -70,12 +70,8 @@ describe('aggregations happy cases', function() {
     ].join(' UNION ALL ');
 
     operations.forEach(function (operation) {
-        var not = operation === 'count' ? ' not ' : ' ';
+        var description = 'should handle NULL values in category and aggregation columns using "' +
-        var description = 'should' +
+            operation + '" as aggregation operation';
-            not +
-            'handle NULL values in category and aggregation columns using "' +
-            operation +
-            '" as aggregation operation';
 
         it(description, function (done) {
             this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val'));
@@ -96,12 +92,7 @@ describe('aggregations happy cases', function() {
                     }
                 });
 
-                if (operation === 'count') {
+                assert.ok(!hasNullCategory, 'aggregation has category NULL');
-                    assert.ok(hasNullCategory, 'aggregation has not a category NULL');
-                } else {
-                    assert.ok(!hasNullCategory, 'aggregation has category NULL');
-                }
-
                 done();
             });
         });
@@ -425,3 +416,79 @@ describe('aggregation dataview tuned by categories query param', function () {
         });
     });
 });
+
+
+
+describe('Count aggregation', function () {
+    const mapConfig = {
+        version: '1.5.0',
+        layers: [
+            {
+                type: "cartodb",
+                options: {
+                    source: {
+                        "id": "a0"
+                    },
+                    cartocss: "#points { marker-width: 10; marker-fill: red; }",
+                    cartocss_version: "2.3.0"
+                }
+            }
+        ],
+        dataviews: {
+            categories: {
+                source: {
+                    id: 'a0'
+                },
+                type: 'aggregation',
+                options: {
+                    column: 'cat',
+                    aggregation: 'count'
+                }
+            }
+        },
+        analyses: [
+            {
+                id: "a0",
+                type: "source",
+                params: {
+                    query: `
+                        SELECT
+                            null::geometry the_geom_webmercator,
+                            CASE
+                                WHEN x % 4 = 0 THEN 1
+                                WHEN x % 4 = 1 THEN 2
+                                WHEN x % 4 = 2 THEN 3
+                                ELSE null
+                            END AS val,
+                            CASE
+                                WHEN x % 4 = 0 THEN 'category_1'
+                                WHEN x % 4 = 1 THEN 'category_2'
+                                WHEN x % 4 = 2 THEN 'category_3'
+                                ELSE null
+                            END AS cat
+                        FROM generate_series(1, 1000) x
+                    `
+                }
+            }
+        ]
+    };
+
+    it(`should handle null values correctly when aggregationColumn isn't provided`, function (done) {
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => {
+            assert.ifError(err);
+            assert.equal(dataview.categories.length, 3);
+            this.testClient.drain(done);
+        });
+    });
+
+    it(`should handle null values correctly when aggregationColumn is provided`, function (done) {
+        mapConfig.dataviews.categories.options.aggregationColumn = 'val';
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => {
+            assert.ifError(err);
+            assert.equal(dataview.categories.length, 3);
+            this.testClient.drain(done);
+        });
+    });
+});

test/acceptance/ported/attributes.js

@@ -125,7 +125,7 @@ describe('attributes', function() {
                     var parsed = JSON.parse(res.body);
                     assert.ok(parsed.errors);
                     var msg = parsed.errors[0];
-                    assert.ok(msg.match(/0 features.*identified by fid -666/), msg);
+                    assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1");
                     return null;
                 },
                 function finish(err) {

test/unit/cartodb/lzmaMiddleware.test.js

@@ -12,6 +12,7 @@ describe('lzma-middleware', function() {
             }
         };
         testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
+            const lzma = lzmaMiddleware();
             var req = {
                 headers: {
                     host:'localhost'
@@ -19,9 +20,13 @@ describe('lzma-middleware', function() {
                 query: {
                     api_key: 'test',
                     lzma: data
+                },
+                profiler: {
+                    done: function () {}
                 }
             };
-            lzmaMiddleware(req, {}, function(err) {
+
+            lzma(req, {}, function(err) {
                 if ( err ) {
                     return done(err);
                 }

test/unit/cartodb/ported/tile_stats.test.js

@@ -1,90 +0,0 @@
-require('../../../support/test_helper.js');
-
-var assert = require('assert');
-
-var LayergroupController = require('../../../../lib/cartodb/controllers/layergroup');
-
-describe('tile stats', function() {
-
-    beforeEach(function () {
-        this.statsClient = global.statsClient;
-    });
-
-    afterEach(function() {
-        global.statsClient = this.statsClient;
-    });
-
-    it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() {
-        var expectedCalls = 2, // it will call increment once for the general error
-            invalidFormat = 'png2',
-            invalidFormatRegexp = new RegExp('invalid'),
-            formatMatched = false;
-        mockStatsClientGetInstance({
-            increment: function(label) {
-                formatMatched = formatMatched || !!label.match(invalidFormatRegexp);
-                expectedCalls--;
-            }
-        });
-
-        var layergroupController = new LayergroupController();
-
-        var reqMock = {
-            profiler: { toJSONString:function() {} },
-            params: {
-                format: invalidFormat
-            }
-        };
-        var resMock = {
-            status: function() { return this; },
-            set: function() {},
-            json: function() {},
-            jsonp: function() {},
-            send: function() {}
-        };
-
-        var next = function () {};
-        layergroupController.finalizeGetTileOrGrid('Unsupported format png2', reqMock, resMock, null, null, next);
-
-        assert.ok(formatMatched, 'Format was never matched in increment method');
-        assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method');
-    });
-
-    it('finalizeGetTileOrGrid calls statsClient when format is supported', function() {
-        var expectedCalls = 2, // general error + format error
-            validFormat = 'png',
-            validFormatRegexp = new RegExp(validFormat),
-            formatMatched = false;
-        mockStatsClientGetInstance({
-            increment: function(label) {
-                formatMatched = formatMatched || !!label.match(validFormatRegexp);
-                expectedCalls--;
-            }
-        });
-        var reqMock = {
-            profiler: { toJSONString:function() {} },
-            params: {
-                format: validFormat
-            }
-        };
-        var resMock = {
-            status: function() { return this; },
-            set: function() {},
-            json: function() {},
-            jsonp: function() {},
-            send: function() {}
-        };
-
-        var layergroupController = new LayergroupController();
-
-        var next = function () {};
-        layergroupController.finalizeGetTileOrGrid('Another error happened', reqMock, resMock, null, null, next);
-
-        assert.ok(formatMatched, 'Format was never matched in increment method');
-        assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method');
-    });
-
-    function mockStatsClientGetInstance(instance) {
-        global.statsClient = Object.assign(global.statsClient, instance);
-    }
-
-});

test/unit/cartodb/prepare-context.test.js

@@ -10,7 +10,7 @@ var TemplateMaps = require('../../../lib/cartodb/backends/template_maps');
 const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/context/clean-up-query-params');
 const authorizeMiddleware = require('../../../lib/cartodb/middleware/context/authorize');
 const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/context/db-conn-setup');
-const apikeyCredentialsMiddleware = require('../../../lib/cartodb/middleware/context/apikey-credentials');
+const credentialsMiddleware = require('../../../lib/cartodb/middleware/context/credentials');
 const localsMiddleware =  require('../../../lib/cartodb/middleware/context/locals');
 
 var windshaft = require('windshaft');
@@ -24,7 +24,7 @@ describe('prepare-context', function() {
     let cleanUpQueryParams;
     let dbConnSetup;
     let authorize;
-    let setApikeyCredentials;
+    let setCredentials;
 
     before(function() {
         var redisPool = new RedisPool(global.environment.redis);
@@ -37,7 +37,7 @@ describe('prepare-context', function() {
         cleanUpQueryParams = cleanUpQueryParamsMiddleware();
         authorize = authorizeMiddleware(authApi);
         dbConnSetup = dbConnSetupMiddleware(pgConnection);
-        setApikeyCredentials = apikeyCredentialsMiddleware();
+        setCredentials = credentialsMiddleware();
     });
 
 
@@ -67,16 +67,17 @@ describe('prepare-context', function() {
     }
 
     it('res.locals are created', function(done) {
+        const locals = localsMiddleware();
         let req = {};
         let res = {};
 
-        localsMiddleware(prepareRequest(req), prepareResponse(res), function(err) {
+        locals(prepareRequest(req), prepareResponse(res), function(err) {
             if ( err ) { done(err); return; }
             assert.ok(res.hasOwnProperty('locals'), 'response has locals');
             done();
         });
     });
-    
+
     it('cleans up request', function(done){
       var req = {headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}};
       var res = {};
@@ -108,18 +109,18 @@ describe('prepare-context', function() {
     });
 
     it('sets also dbuser for authenticated requests', function(done){
-        var req = { 
+        var req = {
-            headers: { 
+            headers: {
-                host: 'localhost' 
+                host: 'localhost'
-            }, 
+            },
             query: {
                 api_key: '1234'
             }
         };
-        var res = { 
+        var res = {
             set: function () {},
             locals: {
-                api_key: '1234' 
+                api_key: '1234'
             }
         };
 
@@ -171,7 +172,7 @@ describe('prepare-context', function() {
             }
         };
         var res = {};
-        
+
         cleanUpQueryParams(prepareRequest(req), prepareResponse(res), function (err) {
             if ( err ) {
                 return done(err);
@@ -196,12 +197,12 @@ describe('prepare-context', function() {
                 }
             };
             var res = {};
-            setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) {
+            setCredentials(prepareRequest(req), prepareResponse(res), function (err) {
                 if (err) {
                     return done(err);
                 }
                 var query = res.locals;
-                
+
                 assert.equal('1234', query.api_key);
                 done();
             });
@@ -217,7 +218,7 @@ describe('prepare-context', function() {
                 }
             };
             var res = {};
-            setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) {
+            setCredentials(prepareRequest(req), prepareResponse(res), function (err) {
                 if (err) {
                     return done(err);
                 }
@@ -236,7 +237,7 @@ describe('prepare-context', function() {
                 }
             };
             var res = {};
-            setApikeyCredentials(prepareRequest(req), prepareResponse(res), function (err) {
+            setCredentials(prepareRequest(req), prepareResponse(res), function (err) {
                 if (err) {
                     return done(err);
                 }

yarn.lock

@@ -11,11 +11,11 @@
     node-pre-gyp "~0.6.30"
     protozero "1.5.1"
 
-"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb1":
+"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb3":
-  version "2.5.1-cdb1"
+  version "2.5.1-cdb3"
-  resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/b0b5559f948e77b337bc9a9ae0bf6ec4249fba21"
+  resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/e61c7752c033595a273dcd1d4b267252b174bd28"
   dependencies:
-    "@carto/mapnik" "~3.6.2-carto.0"
+    "@carto/mapnik" "3.6.2-carto.2"
     "@mapbox/sphericalmercator" "~1.0.1"
     mapnik-pool "~0.1.3"
 
@@ -23,7 +23,7 @@
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2"
 
-abaculus@cartodb/abaculus#2.0.3-cdb2:
+"abaculus@github:cartodb/abaculus#2.0.3-cdb2":
   version "2.0.3-cdb2"
   resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc"
   dependencies:
@@ -249,7 +249,7 @@ camshaft@0.61.2:
     dot "^1.0.3"
     request "^2.69.0"
 
-canvas@cartodb/node-canvas#1.6.2-cdb2:
+"canvas@github:cartodb/node-canvas#1.6.2-cdb2":
   version "1.6.2-cdb2"
   resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
   dependencies:
@@ -275,7 +275,7 @@ carto@CartoDB/carto#0.15.1-cdb1:
     optimist "~0.6.0"
     underscore "~1.6.0"
 
-carto@cartodb/carto#0.15.1-cdb3:
+"carto@github:cartodb/carto#0.15.1-cdb3":
   version "0.15.1-cdb3"
   resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
   dependencies:
@@ -2212,11 +2212,11 @@ through@2:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
 
-tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb5:
+"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb7":
-  version "0.6.18-cdb5"
+  version "0.6.18-cdb7"
-  resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cec846025e60837c60af193d600d972917ea8d35"
+  resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/488d2acd65c89cc5382d996eabe8dc1f5051ce0f"
   dependencies:
-    "@carto/mapnik" "~3.6.2-carto.0"
+    "@carto/mapnik" "3.6.2-carto.2"
     generic-pool "~2.4.0"
     mime "~1.6.0"
     sphericalmercator "~1.0.4"
@@ -2373,12 +2373,12 @@ window-size@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
 
-windshaft@4.5.3:
+windshaft@4.5.4:
-  version "4.5.3"
+  version "4.5.4"
-  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.3.tgz#45e792af06b224f78f44b6eb3b0ecb9d90dcc943"
+  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.4.tgz#ce9b2f1bbc8ef749a26693e1832774b438d78cb9"
   dependencies:
     "@carto/mapnik" "3.6.2-carto.2"
-    "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb1
+    "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb3
     abaculus cartodb/abaculus#2.0.3-cdb2
     canvas cartodb/node-canvas#1.6.2-cdb2
     carto cartodb/carto#0.15.1-cdb3
@@ -2393,7 +2393,7 @@ windshaft@4.5.3:
     sphericalmercator "1.0.4"
     step "~0.0.6"
     tilelive "5.12.2"
-    tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb5
+    tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb7
     torque.js "~2.11.0"
     underscore "~1.6.0"