Merge pull request #1111 from Algunenano/cartodbless

Render MVTs and aggregations without cartodb-postgresql

lib/cartodb/backends/cluster.js

@@ -5,6 +5,9 @@ const dbParamsFromReqParams = require('../utils/database-params');
 const debug = require('debug')('backend:cluster');
 const AggregationMapConfig = require('../models/aggregation/aggregation-mapconfig');
 
+const windshaftUtils = require('windshaft').utils;
+const webmercator = new windshaftUtils.WebMercatorHelper();
+
 module.exports = class ClusterBackend {
     getClusterFeatures (mapConfigProvider, params, callback) {
         mapConfigProvider.getMapConfig((err, _mapConfig) => {
@@ -127,7 +130,7 @@ function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, ag
     } , true); // use read-only transaction
 }
 
-const schemaQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
+const schemaQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_cluster_schema LIMIT 0`;
 const clusterFeaturesQuery = ctx => `
     WITH
     _cdb_params AS (
@@ -156,9 +159,8 @@ const clusterFeaturesQuery = ctx => `
 `;
 
 const gridResolution = ctx => {
-    const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38);
+    const zoomResolution = webmercator.getResolution({ z : Math.min(38, ctx.zoom) });
-    const pixelSize = `CDB_XYZ_Resolution(${ctx.zoom})`;
+    return `${256/ctx.res} * (${zoomResolution})::double precision`;
-    return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
 };
 
 const aggregationQuery = ctx => `

lib/cartodb/models/aggregation/aggregation-mapconfig.js

@@ -201,7 +201,7 @@ module.exports = class AggregationMapConfig extends MapConfig {
 
     getLayerColumns (index, skipGeoms, callback) {
         const geomColumns = ['the_geom', 'the_geom_webmercator'];
-        const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
+        const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_aggregation_schema LIMIT 0`;
         const layer = this.getLayer(index);
 
         this.pgConnection.getConnection(this.user, (err, connection) => {

lib/cartodb/models/aggregation/aggregation-query.js

@@ -3,31 +3,8 @@
 const timeDimension = require('./time-dimension');
 
 const DEFAULT_PLACEMENT = 'point-sample';
-
+const windshaftUtils = require('windshaft').utils;
-
+const webmercator = new windshaftUtils.WebMercatorHelper();
-/**
- * Returns a template function (function that accepts template parameters and returns a string)
- * to generate an aggregation query.
- * Valid options to define the query template are:
- * - placement
- * - columns
- * - dimensions*
- * The query template parameters taken by the result template function are:
- * - sourceQuery
- * - res
- * - columns
- * - dimensions
- */
-const templateForOptions = (options) => {
-    let templateFn = defaultAggregationQueryTemplate;
-    if (!options.isDefaultAggregation) {
-        templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT];
-        if (!templateFn) {
-            throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
-        }
-    }
-    return templateFn;
-};
 
 function optionsToParams (options) {
     return {
@@ -35,7 +12,9 @@ function optionsToParams (options) {
         res: 256/options.resolution,
         columns: options.columns,
         dimensions: options.dimensions,
-        filters: options.filters
+        filters: options.filters,
+        placement: options.placement || DEFAULT_PLACEMENT,
+        isDefaultAggregation: options.isDefaultAggregation
     };
 }
 
@@ -53,7 +32,7 @@ function optionsToParams (options) {
  * When placement, columns or dimensions are specified, columns are aggregated as requested
  * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
  */
-const queryForOptions = (options) => templateForOptions(options)(optionsToParams(options));
+const queryForOptions = (options) => aggregationQueryTemplate(optionsToParams(options));
 
 module.exports = queryForOptions;
 
@@ -87,7 +66,7 @@ const SUPPORTED_AGGREGATE_FUNCTIONS = {
         sql: (column_name, params) => `max(${params.aggregated_column || column_name})`
     },
     'mode': {
-        sql: (column_name, params) => `_cdb_mode(${params.aggregated_column || column_name})`
+        sql: (column_name, params) => `mode() WITHIN GROUP (ORDER BY ${params.aggregated_column || column_name})`
     }
 };
 
@@ -106,16 +85,6 @@ const aggregateColumns = ctx => {
     }, ctx.columns || {});
 };
 
-const aggregateColumnNames = (ctx, table) => {
-    let columns = aggregateColumns(ctx);
-    if (table) {
-        return sep(Object.keys(columns).map(
-            column_name => `${table}.${column_name}`
-        ));
-    }
-    return sep(Object.keys(columns));
-};
-
 const aggregateExpression = (column_name, column_parameters) => {
     const aggregate_function = column_parameters.aggregate_function || 'count';
     const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
@@ -297,156 +266,158 @@ const havingClause = ctx => {
 // (i.e. each tile is divided into ctx.res*ctx.res cells).
 // We limit the the minimum resolution to avoid division by zero problems. The limit used is
 // the pixel size of zoom level 30 (i.e. 1/2*(30+8) of the full earth web-mercator extent), which is about 0.15 mm.
-// Computing this using !scale_denominator!, !pixel_width! or !pixel_height! produces
+//
-// inaccurate results due to rounding present in those values.
+// NOTE: We'd rather use !pixel_width!, but in Mapnik this value is extent / 256 for raster
+// and extent / tile_extent {4096 default} for MVT, so since aggregations are always based
+// on 256 we can't have the same query in both cases
+// As this scale change doesn't happen in !scale_denominator! we use that instead
+// NOTE 2: The 0.00028 is used in Mapnik (and replicated in pg-mvt) and comes from
+// OGC's Styled Layer Descriptor Implementation Specification
 const gridResolution = ctx => {
-    const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38);
+    const minimumResolution = webmercator.getResolution({ z : 38 });
-    const pixelSize = 'CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))';
+    return `${256/ctx.res} * GREATEST(!scale_denominator! * 0.00028, ${minimumResolution})::double precision`;
-    return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
 };
 
-// Each aggregation cell is defined by the cell coordinates Floor(x/res), Floor(y/res),
+// SQL query to extract the boundaries of the area to be aggregated and the grid resolution
-// i.e. they include the West and South borders but not the East and North ones.
+// cdb_{x-y}{min_max} return the limits of the tile. Aggregations do [min, max) in both axis
-// So, to avoid picking points that don't belong to cells in the tile, given the tile
+// cdb_res: Aggregation resolution (as specified by gridResolution)
-// limits Xmin, Ymin, Xmax, Ymax (bbox), we should select points that satisfy
+// cdb_point_bbox: Tile bounding box [min, max]
-// Xmin <= x < Xmax and Ymin <= y < Ymax (with x, y from the_geom_webmercator)
+const gridInfoQuery = ctx => {
-// On the other hand we can efficiently filter spatially (relying on spatial indexing)
+    return `
-// with `the_geom_webmercator && bbox` which is equivalent to
-//   Xmin <= x <= Xmax and Ymin <= y <= Ymax
-// So, in order to be both efficient and accurate we will need to use both
-// conditions for spatial filtering.
-const spatialFilter = `
-  (_cdb_query.the_geom_webmercator && _cdb_params.bbox) AND
-    ST_X(_cdb_query.the_geom_webmercator) >= _cdb_params.xmin AND
-    ST_X(_cdb_query.the_geom_webmercator) < _cdb_params.xmax AND
-    ST_Y(_cdb_query.the_geom_webmercator) >= _cdb_params.ymin AND
-    ST_Y(_cdb_query.the_geom_webmercator) < _cdb_params.ymax
-`;
-
-// Notes:
-// * We need to filter spatially using !bbox! to make the queries efficient because
-//   the filter added by Mapnik (wrapping the query)
-//   is only applied after the aggregation.
-// * This queries are used for rendering and the_geom is omitted in the results for better performance
-// * If the MVT extent or tile buffer was 0 or a multiple of the resolution we could use directly
-//   the bbox for them, but in general we need to find the nearest cell limits inside the bbox.
-// * bbox coordinates can have an error in the last digits; we apply a small correction before
-//   applying CEIL or FLOOR to compensate for this, so that coordinates closer than a small (`eps`)
-//   fraction of the cell size to a cell limit are moved to the exact limit.
-const sqlParams = (ctx) => `
-_cdb_res AS (
     SELECT
-    ${gridResolution(ctx)} AS res,
+        cdb_xmin,
-    !bbox! AS bbox,
+        cdb_ymin,
-    (1E-6::double precision) AS eps
+        cdb_xmax,
-),
+        cdb_ymax,
-_cdb_params AS (
+        cdb_res,
-    SELECT
+        ST_MakeEnvelope(cdb_xmin, cdb_ymin, cdb_xmax, cdb_ymax, 3857) AS cdb_point_bbox
-      res,
-      bbox,
-      CEIL((ST_XMIN(bbox) - eps*res)/res)*res AS xmin,
-      FLOOR((ST_XMAX(bbox) + eps*res)/res)*res AS xmax,
-      CEIL((ST_YMIN(bbox) - eps*res)/res)*res AS ymin,
-      FLOOR((ST_YMAX(bbox) + eps*res)/res)*res AS ymax
-      FROM _cdb_res
-)
-`;
-
-// The special default aggregation includes all the columns of a sample row per grid cell and
-// the count (_cdb_feature_count) of the aggregated rows.
-const defaultAggregationQueryTemplate = ctx => `
-    WITH ${sqlParams(ctx)},
-    _cdb_clusters AS (
-        SELECT
-            MIN(cartodb_id) AS cartodb_id
-            ${dimensionDefs(ctx)}
-            ${aggregateColumnDefs(ctx)}
-        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
-        WHERE ${spatialFilter}
-        GROUP BY
-            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
-            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
-            ${dimensionNames(ctx)}
-    ) SELECT
-        _cdb_query.*
-        ${aggregateColumnNames(ctx)}
     FROM
-        _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
+    (
-        ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
-`;
-
-const aggregationQueryTemplates = {
-    'centroid': ctx => `
-        WITH ${sqlParams(ctx)}
         SELECT
-            MIN(_cdb_query.cartodb_id)  AS cartodb_id,
+            cdb_res,
-            ST_SetSRID(
+            CEIL (ST_XMIN(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_xmin,
-                ST_MakePoint(
+            FLOOR(ST_XMAX(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_xmax,
-                    AVG(ST_X(_cdb_query.the_geom_webmercator)),
+            CEIL (ST_YMIN(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_ymin,
-                    AVG(ST_Y(_cdb_query.the_geom_webmercator))
+            FLOOR(ST_YMAX(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_ymax
-                ), 3857
-            ) AS the_geom_webmercator
-            ${dimensionDefs(ctx)}
-            ${aggregateColumnDefs(ctx)}
-        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
-        WHERE ${spatialFilter}
-        GROUP BY
-            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
-            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
-            ${dimensionNames(ctx)}
-        ${havingClause(ctx)}
-    `,
-
-    'point-grid': ctx => `
-        WITH ${sqlParams(ctx)},
-        _cdb_clusters AS (
-            SELECT
-            MIN(_cdb_query.cartodb_id) AS cartodb_id,
-            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
-            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy
-            ${dimensionDefs(ctx)}
-            ${aggregateColumnDefs(ctx)}
-            FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
-            WHERE ${spatialFilter}
-            GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
-            ${havingClause(ctx)}
-        )
-        SELECT
-            _cdb_clusters.cartodb_id AS cartodb_id,
-            ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator
-            ${dimensionNames(ctx)}
-            ${aggregateColumnNames(ctx)}
-            FROM _cdb_clusters, _cdb_params
-    `,
-
-    'point-sample': ctx => `
-        WITH ${sqlParams(ctx)},
-        _cdb_clusters AS (
-            SELECT
-                MIN(cartodb_id) AS cartodb_id
-                ${dimensionDefs(ctx)}
-                ${aggregateColumnDefs(ctx)}
-            FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
-            WHERE ${spatialFilter}
-            GROUP BY
-                Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
-                Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
-                ${dimensionNames(ctx)}
-            ${havingClause(ctx)}
-        )
-        SELECT
-            _cdb_clusters.cartodb_id,
-            the_geom_webmercator
-            ${dimensionNames(ctx, '_cdb_clusters')}
-            ${aggregateColumnNames(ctx, '_cdb_clusters')}
         FROM
-            _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
+        (
-            ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
+            SELECT
-    `
+                ${gridResolution(ctx)} AS cdb_res,
-
+                !bbox! cdb_full_bbox
+        ) _cdb_input_resources
+    ) _cdb_grid_bbox_margins
+`;
 };
 
-module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates);
+
+// Function to generate the resulting point for a cell from the aggregated data
+const aggregatedPointWebMercator = (ctx) => {
+    switch (ctx.placement) {
+
+        // For centroid, we return the average of the cell
+        case 'centroid':
+            return ', ST_SetSRID(ST_MakePoint(AVG(cdb_x), AVG(cdb_y)), 3857) AS the_geom_webmercator';
+
+        // Middle point of the cell
+        case 'point-grid':
+            return `, ST_SetSRID(ST_MakePoint(cdb_pos_grid_x, cdb_pos_grid_y), 3857) AS the_geom_webmercator`;
+
+        // For point-sample we'll get a single point directly from the source
+        // If it's default aggregation we'll add the extra columns to keep backwards compatibility
+        case 'point-sample':
+            return '';
+
+        default:
+            throw new Error(`Invalid aggregation placement "${ctx.placement}"`);
+    }
+};
+
+// Function to generate the resulting point for a cell from the a join with the source
+const aggregatedPointJoin = (ctx) => {
+    switch (ctx.placement) {
+
+        case 'centroid':
+            return '';
+
+        case 'point-grid':
+            return '';
+
+        // For point-sample we'll get a single point directly from the source
+        // If it's default aggregation we'll add the extra columns to keep backwards compatibility
+        case 'point-sample':
+            return `
+NATURAL JOIN
+(
+    SELECT ${ctx.isDefaultAggregation ? `*` : `cartodb_id, the_geom_webmercator`}
+    FROM
+    (
+        ${ctx.sourceQuery}
+    ) __cdb_src_query
+) __cdb_query_columns
+`;
+
+        default:
+            throw new Error('Invalid aggregation placement "${ctx.placement}"');
+    }
+};
+
+// Function to generate the values common to all points in a cell
+// By default we use the cell number (which is fast), but for point-grid we
+// get the coordinates of the mid point so we don't need to calculate them later
+// which requires extra data in the group by clause
+const aggregatedPosCoordinate = (ctx, coordinate) => {
+    switch (ctx.placement) {
+        // For point-grid we return the coordinate of the middle point of the grid
+        case `point-grid`:
+            return `(FLOOR(cdb_${coordinate} / __cdb_grid_params.cdb_res) + 0.5) * __cdb_grid_params.cdb_res`;
+
+        // For other, we return the cell position (relative to the world)
+        default:
+            return `FLOOR(cdb_${coordinate} / __cdb_grid_params.cdb_res)`;
+    }
+};
+
+
+const aggregationQueryTemplate = ctx => `
+WITH __cdb_grid_params AS
+(
+    ${gridInfoQuery(ctx)}
+)
+SELECT * FROM
+(
+    SELECT
+        min(cartodb_id) as cartodb_id
+        ${aggregatedPointWebMercator(ctx)}
+        ${dimensionDefs(ctx)}
+        ${aggregateColumnDefs(ctx)}
+    FROM
+    (
+        SELECT
+            *,
+            ${aggregatedPosCoordinate(ctx, 'x')} as cdb_pos_grid_x,
+            ${aggregatedPosCoordinate(ctx, 'y')} as cdb_pos_grid_y
+        FROM
+        (
+            SELECT
+                __cdb_src_query.*,
+                ST_X(the_geom_webmercator) cdb_x,
+                ST_Y(the_geom_webmercator) cdb_y
+            FROM
+            (
+                ${ctx.sourceQuery}
+            ) __cdb_src_query, __cdb_grid_params
+            WHERE the_geom_webmercator && cdb_point_bbox
+            OFFSET 0
+        ) __cdb_src_get_x_y, __cdb_grid_params
+        WHERE cdb_x < __cdb_grid_params.cdb_xmax AND cdb_y < __cdb_grid_params.cdb_ymax
+    ) __cdb_src_gridded
+    GROUP BY cdb_pos_grid_x, cdb_pos_grid_y ${dimensionNames(ctx)}
+    ${havingClause(ctx)}
+) __cdb_aggregation_src
+${aggregatedPointJoin(ctx)}
+`;
+
+module.exports.SUPPORTED_PLACEMENTS = ['centroid', 'point-grid', 'point-sample'];
 module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator';
 
 const clusterFeaturesQuery = ctx => `

lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -2,6 +2,7 @@
 
 var queue = require('queue-async');
 var _ = require('underscore');
+const AggregationMapConfig = require('../../aggregation/aggregation-mapconfig');
 
 function MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend) {
     this.overviewsMetadataBackend = overviewsMetadataBackend;
@@ -14,7 +15,9 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function (user, requestMapCon
     var layers = requestMapConfig.layers;
     var analysesResults = context.analysesResults;
 
-    if (!layers || layers.length === 0) {
+    const aggMapConfig = new AggregationMapConfig(null, requestMapConfig);
+    if (aggMapConfig.isVectorOnlyMapConfig() || aggMapConfig.isAggregationMapConfig() ||
+        !layers || layers.length === 0) {
         return callback(null, requestMapConfig);
     }
 

lib/cartodb/utils/query-utils.js

@@ -1,48 +1,41 @@
 'use strict';
 
-const SubstitutionTokens = require('./substitution-tokens');
+const windshaftUtils = require('windshaft').utils;
 
-function prepareQuery(sql) {
+module.exports.extractTableNames = function (query) {
-    var affectedTableRegexCache = {
-        bbox: /!bbox!/g,
-        scale_denominator: /!scale_denominator!/g,
-        pixel_width: /!pixel_width!/g,
-        pixel_height: /!pixel_height!/g
-    };
-
-    return sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.scale_denominator, '0')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1');
-}
-
-module.exports.extractTableNames = function extractTableNames(query) {
     return [
         'SELECT * FROM CDB_QueryTablesText($windshaft$',
-        prepareQuery(query),
+        substituteDummyTokens(query),
         '$windshaft$) as tablenames'
     ].join('');
 };
 
 module.exports.getQueryActualRowCount = function (query) {
-    return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`;
+    return `select COUNT(*) AS rows FROM (${substituteDummyTokens(query)}) AS __cdb_query`;
 };
 
 function getQueryRowEstimation(query) {
-    return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows';
+    return 'select CDB_EstimateRowCount($windshaft$' + substituteDummyTokens(query) + '$windshaft$) as rows';
 }
-
 module.exports.getQueryRowEstimation = getQueryRowEstimation;
 
+function getQueryGeometryType(query, geometryColumn) {
+    return `
+        SELECT ST_GeometryType(${geometryColumn}) AS geom_type
+            FROM (${substituteDummyTokens(query)}) AS __cdb_query
+            WHERE ${geometryColumn} IS NOT NULL
+            LIMIT 1
+    `;
+}
+module.exports.getQueryGeometryType = getQueryGeometryType;
+
 module.exports.getAggregationMetadata = ctx => `
     WITH
     rowEstimation AS (
         ${getQueryRowEstimation(ctx.query)}
     ),
     geometryType AS (
-        SELECT ST_GeometryType(${ctx.geometryColumn}) as geom_type
+        ${getQueryGeometryType(ctx.query, ctx.geometryColumn)}
-        FROM (${ctx.query}) AS __cdb_query WHERE ${ctx.geometryColumn} IS NOT NULL LIMIT 1
     )
     SELECT
         rows AS count,
@@ -85,7 +78,7 @@ module.exports.getQueryTopCategories = function (query, column, topN, includeNul
     const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`;
     return `
         SELECT ${column} AS category, COUNT(*) AS frequency
-        FROM (${query}) AS __cdb_query
+        FROM (${substituteDummyTokens(query)}) AS __cdb_query
         ${where}
         GROUP BY ${column} ORDER BY 2 DESC
         LIMIT ${topN}
@@ -116,7 +109,7 @@ module.exports.getQuerySample = function (query, sampleProb, limit = null, rando
             SELECT setseed(${randomSeed})
         )
         SELECT ${columnSelector(columns)}
-            FROM (${query}) AS __cdb_query
+            FROM (${substituteDummyTokens(query)}) AS __cdb_query
             WHERE random() < ${sampleProb}
             ${limitClause}
     `;
@@ -157,19 +150,11 @@ function simpleQueryTable(sql) {
     return false;
 }
 
-module.exports.getQueryGeometryType = function (query, geometryColumn) {
-    return `
-        SELECT ST_GeometryType(${geometryColumn}) AS geom_type
-            FROM (${query}) AS __cdb_query
-            WHERE ${geometryColumn} IS NOT NULL
-            LIMIT 1
-    `;
-};
 
 function getQueryLimited(query, limit = 0) {
     return `
         SELECT *
-            FROM (${query}) AS __cdb_query
+            FROM (${substituteDummyTokens(query)}) AS __cdb_query
             LIMIT ${limit}
     `;
 }
@@ -181,32 +166,32 @@ function queryPromise(dbConnection, query) {
 }
 
 function substituteDummyTokens(sql) {
-    return sql && SubstitutionTokens.replace(sql, {
+    return subsituteTokensForZoom(sql, 0);
-        bbox: 'ST_MakeEnvelope(0,0,0,0)',
-        scale_denominator: '0',
-        pixel_width: '1',
-        pixel_height: '1'
-    });
 }
 
-function subsituteTokensForZoom(sql, zoom, singleTile=false) {
+function subsituteTokensForZoom(sql, zoom) {
-    const tileRes = 256;
+    if (!sql) {
-    const wmSize = 6378137.0*2*Math.PI;
+        return undefined;
-    const nTiles = Math.pow(2, zoom);
-    const tileSize = wmSize / nTiles;
-    const resolution = tileSize / tileRes;
-    const scaleDenominator = resolution / 0.00028;
-    const x0 = -wmSize/2, y0 = -wmSize/2;
-    let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`;
-    if (singleTile) {
-        bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`;
     }
-    return SubstitutionTokens.replace(sql, {
+    const affectedTableRegexCache = {
-        bbox: bbox,
+        bbox: /!bbox!/g,
-        scale_denominator: scaleDenominator,
+        scale_denominator: /!scale_denominator!/g,
-        pixel_width: resolution,
+        pixel_width: /!pixel_width!/g,
-        pixel_height: resolution
+        pixel_height: /!pixel_height!/g
-    });
+    };
+
+    const webmercator = new windshaftUtils.WebMercatorHelper();
+    const resolution = webmercator.getResolution({ z : zoom });
+    const scaleDenominator = resolution.dividedBy(0.00028);
+    // We always use the whole world as the bbox
+    const extent = webmercator.getExtent({ x : 0, y : 0, z : 0 });
+
+    return sql
+        .replace(affectedTableRegexCache.bbox,
+                 `ST_MakeEnvelope(${extent.xmin}, ${extent.ymin}, ${extent.xmax}, ${extent.ymax}, 3857)`)
+        .replace(affectedTableRegexCache.scale_denominator, scaleDenominator)
+        .replace(affectedTableRegexCache.pixel_width, resolution)
+        .replace(affectedTableRegexCache.pixel_height, resolution);
 }
 
 module.exports.queryPromise = queryPromise;

test/acceptance/aggregation.js

@@ -6,6 +6,9 @@ const assert = require('../support/assert');
 const TestClient = require('../support/test-client');
 const serverOptions = require('../../lib/cartodb/server_options');
 
+const windshaftUtils = require('windshaft').utils;
+const webmercator = new windshaftUtils.WebMercatorHelper();
+
 const suites = [
     {
         desc: 'mvt (mapnik)',
@@ -96,9 +99,9 @@ describe('aggregation', function () {
     const POLYGONS_SQL_1 = `
     select
         x + 4 as cartodb_id,
-        st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 100000)::geometry as the_geom,
+        st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326), 10) as the_geom,
         st_transform(
-            st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 100000)::geometry,
+            st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326), 10),
             3857
         ) as the_geom_webmercator,
         x as value
@@ -109,15 +112,15 @@ describe('aggregation', function () {
     WITH hgrid AS (
         SELECT
             CDB_RectangleGrid (
-                ST_Expand(!bbox!, CDB_XYZ_Resolution(1) * 12),
+                ST_Expand(!bbox!, ${webmercator.getResolution({ z : 1 })} * 12),
-                CDB_XYZ_Resolution(1) * 12,
+                ${webmercator.getResolution({ z : 1 })} * 12,
-                CDB_XYZ_Resolution(1) * 12
+                ${webmercator.getResolution({ z : 1 })} * 12
             ) as cell
     )
     SELECT
         hgrid.cell as the_geom_webmercator,
         count(1) as agg_value,
-        count(1) /power( 12 * CDB_XYZ_Resolution(1), 2 ) as agg_value_density,
+        count(1) /power( 12 * ${webmercator.getResolution({ z : 1 })}, 2 ) as agg_value_density,
         row_number() over () as cartodb_id
     FROM hgrid, (<%= sql %>) i
     WHERE ST_Intersects(i.the_geom_webmercator, hgrid.cell) GROUP BY hgrid.cell
@@ -202,9 +205,9 @@ describe('aggregation', function () {
     //       |   |   |   |   |   |   |
     //    Tile 0, 1 -+---+---+---+- Tile 1,1
     //
-    const POINTS_SQL_GRID = `
+    const POINTS_SQL_GRID = (z, resolution) => `
         WITH params AS (
-            SELECT CDB_XYZ_Resolution($Z)*$resolution AS l -- cell size for Z, resolution
+            SELECT ${webmercator.getResolution({ z : z })}*${resolution} AS l -- cell size for Z, resolution
         )
         SELECT
             row_number() OVER () AS cartodb_id,
@@ -224,8 +227,8 @@ describe('aggregation', function () {
     const POINTS_SQL_CELL = `
       SELECT
         1 AS cartodb_id,
-        ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857) AS the_geom_webmercator,
+        ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857) AS the_geom_webmercator,
-        ST_Transform(ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857), 4326) AS the_geom
+        ST_Transform(ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857), 4326) AS the_geom
       UNION ALL SELECT
         2 AS cartodb_id,
         ST_SetSRID(ST_MakePoint(18181005.9, -18181044.0), 3857) AS the_geom_webmercator,
@@ -236,8 +239,8 @@ describe('aggregation', function () {
         ST_Transform(ST_SetSRID(ST_MakePoint(18181005.87, -18181043.94), 3857), 4326) AS the_geom
       UNION ALL SELECT
         4 AS cartodb_id,
-        ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857) AS the_geom_webmercator,
+        ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857) AS the_geom_webmercator,
-        ST_Transform(ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857), 4326) AS the_geom
+        ST_Transform(ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857), 4326) AS the_geom
     `;
 
     // Points positioned inside one cell of Z=20, X=1000000, X=1000000 (inner cell not on border)
@@ -249,8 +252,8 @@ describe('aggregation', function () {
         ST_Transform(ST_SetSRID(ST_MakePoint(18181005.95, -18181043.8), 3857), 4326) AS the_geom
       UNION ALL SELECT
         2 AS cartodb_id,
-        ST_SetSRID(ST_MakePoint(18181006.09, -18181043.72), 3857) AS the_geom_webmercator,
+        ST_SetSRID(ST_MakePoint(18181006.09, -18181043.74), 3857) AS the_geom_webmercator,
-        ST_Transform(ST_SetSRID(ST_MakePoint(18181006.09, -18181043.72), 3857), 4326) AS the_geom
+        ST_Transform(ST_SetSRID(ST_MakePoint(18181006.09, -18181043.74), 3857), 4326) AS the_geom
       UNION ALL SELECT
         3 AS cartodb_id,
         ST_SetSRID(ST_MakePoint(18181006.02, -18181043.79), 3857) AS the_geom_webmercator,
@@ -521,7 +524,7 @@ describe('aggregation', function () {
             });
 
             ['centroid', 'point-sample', 'point-grid'].forEach(placement => {
-                it('should provide all the requested columns in non-default aggregation ',
+                it('should provide all the requested columns in non-default aggregation: ' + placement,
                 function (done) {
                     const response = {
                         status: 200,
@@ -570,7 +573,7 @@ describe('aggregation', function () {
                     });
                 });
 
-                it('should provide only the requested columns in non-default aggregation ',
+                it('should provide only the requested columns in non-default aggregation: ' + placement,
                 function (done) {
                     this.mapConfig = createVectorMapConfig([
                         {
@@ -2351,7 +2354,7 @@ describe('aggregation', function () {
                                     threshold: 1,
                                     columns: {
                                         value: {
-                                            aggregate_function: 'sum',
+                                            aggregate_function: 'mode',
                                             aggregated_column: 'value'
                                         }
                                     },
@@ -2397,7 +2400,7 @@ describe('aggregation', function () {
                                     threshold: 1,
                                     columns: {
                                         value: {
-                                            aggregate_function: 'sum',
+                                            aggregate_function: 'mode',
                                             aggregated_column: 'value'
                                         }
                                     },
@@ -2970,7 +2973,7 @@ describe('aggregation', function () {
                 it(`for ${placement} each aggr. cell is in a single tile`, function (done) {
                     const z = 1;
                     const resolution = 1;
-                    const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution);
+                    const query = POINTS_SQL_GRID(z, resolution);
                     this.mapConfig = {
                         version: '1.6.0',
                         buffersize: { 'mvt': 0 },
@@ -3015,32 +3018,17 @@ describe('aggregation', function () {
                                     }
                                     const tile11 = JSON.parse(mvt.toGeoJSONSync(0));
 
-                                    const tile00Expected = [
+                                    // There needs to be 13 points
-                                        { cartodb_id: 4, _cdb_feature_count: 2 },
+                                    const count_features = ((tile) =>
-                                        { cartodb_id: 7, _cdb_feature_count: 1 }
+                                        tile.features.map(f => f.properties)
-                                    ];
+                                                     .map(f => f._cdb_feature_count)
-                                    const tile10Expected = [
+                                                     .reduce((a,b) => a + b, 0));
-                                        { cartodb_id: 5, _cdb_feature_count: 2 },
+
-                                        { cartodb_id: 6, _cdb_feature_count: 1 },
+                                    const tile00Count = count_features(tile00);
-                                        { cartodb_id: 8, _cdb_feature_count: 1 },
+                                    const tile10Count = count_features(tile10);
-                                        { cartodb_id: 9, _cdb_feature_count: 1 }
+                                    const tile01Count = count_features(tile01);
-                                    ];
+                                    const tile11Count = count_features(tile11);
-                                    const tile01Expected = [
+                                    assert.equal(13, tile00Count + tile10Count + tile01Count + tile11Count);
-                                        { cartodb_id: 1, _cdb_feature_count: 2 }
-                                    ];
-                                    const tile11Expected = [
-                                        { cartodb_id: 2, _cdb_feature_count: 2 },
-                                        { cartodb_id: 3, _cdb_feature_count: 1 }
-                                    ];
-                                    const tile00Actual = tile00.features.map(f => f.properties);
-                                    const tile10Actual = tile10.features.map(f => f.properties);
-                                    const tile01Actual = tile01.features.map(f => f.properties);
-                                    const tile11Actual = tile11.features.map(f => f.properties);
-                                    const orderById = (a, b) => a.cartodb_id - b.cartodb_id;
-                                    assert.deepEqual(tile00Actual.sort(orderById), tile00Expected);
-                                    assert.deepEqual(tile10Actual.sort(orderById), tile10Expected);
-                                    assert.deepEqual(tile01Actual.sort(orderById), tile01Expected);
-                                    assert.deepEqual(tile11Actual.sort(orderById), tile11Expected);
 
                                     done();
                                 });
@@ -3056,7 +3044,7 @@ describe('aggregation', function () {
                     const z = 1;
                     const resolution = 2;
                     // space the test points by half the resolution:
-                    const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution/2);
+                    const query = POINTS_SQL_GRID(z, resolution / 2);
 
                     this.mapConfig = {
                         version: '1.6.0',
@@ -3133,20 +3121,11 @@ describe('aggregation', function () {
                 });
 
                 it(`for ${placement} includes complete cells in buffer`, function (done) {
-                    if (!usePostGIS && placement !== 'point-grid') {
-                        // Mapnik seem to filter query results by its (inaccurate) bbox,
-                        // which makes some aggregated clusters get lost here.
-                        // The point-grid placement is resilient to this problem because the result
-                        // coordinates are moved to cluster cell centers, so they're well within
-                        // bbox limits.
-                        this.testClient = new TestClient({});
-                        return done();
-                    }
 
                     // use buffersize coincident with resolution, the buffer should include neighbour cells
                     const z = 2;
                     const resolution = 1;
-                    const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution);
+                    const query = POINTS_SQL_GRID(z, resolution);
 
                     this.mapConfig = {
                         version: '1.6.0',
@@ -3192,49 +3171,24 @@ describe('aggregation', function () {
                                     }
                                     const tile11 = JSON.parse(mvt.toGeoJSONSync(0));
 
-                                    const tile00Expected = [
+                                    // We check that if an id/cell is present in multiple tiles,
-                                        { _cdb_feature_count: 2, cartodb_id: 1 },
+                                    // it always contains the same amount of features
-                                        { _cdb_feature_count: 2, cartodb_id: 2 },
-                                        { _cdb_feature_count: 2, cartodb_id: 4 },
-                                        { _cdb_feature_count: 2, cartodb_id: 5 },
-                                        { _cdb_feature_count: 1, cartodb_id: 7 },
-                                        { _cdb_feature_count: 1, cartodb_id: 8 }
-                                    ];
-                                    const tile10Expected = [
-                                        { _cdb_feature_count: 2, cartodb_id: 1 },
-                                        { _cdb_feature_count: 2, cartodb_id: 2 },
-                                        { _cdb_feature_count: 1, cartodb_id: 3 },
-                                        { _cdb_feature_count: 2, cartodb_id: 4 },
-                                        { _cdb_feature_count: 2, cartodb_id: 5 },
-                                        { _cdb_feature_count: 1, cartodb_id: 6 },
-                                        { _cdb_feature_count: 1, cartodb_id: 7 },
-                                        { _cdb_feature_count: 1, cartodb_id: 8 },
-                                        { _cdb_feature_count: 1, cartodb_id: 9 }
-                                    ];
-                                    const tile01Expected = [
-                                        { _cdb_feature_count: 2, cartodb_id: 1 },
-                                        { _cdb_feature_count: 2, cartodb_id: 2 },
-                                        { _cdb_feature_count: 2, cartodb_id: 4 },
-                                        { _cdb_feature_count: 2, cartodb_id: 5 }
-                                    ];
-                                    const tile11Expected = [
-                                        { _cdb_feature_count: 2, cartodb_id: 1 },
-                                        { _cdb_feature_count: 2, cartodb_id: 2 },
-                                        { _cdb_feature_count: 1, cartodb_id: 3 },
-                                        { _cdb_feature_count: 2, cartodb_id: 4 },
-                                        { _cdb_feature_count: 2, cartodb_id: 5 },
-                                        { _cdb_feature_count: 1, cartodb_id: 6 }
-                                    ];
                                     const tile00Actual = tile00.features.map(f => f.properties);
                                     const tile10Actual = tile10.features.map(f => f.properties);
                                     const tile01Actual = tile01.features.map(f => f.properties);
                                     const tile11Actual = tile11.features.map(f => f.properties);
-                                    const orderById = (a, b) => a.cartodb_id - b.cartodb_id;
-                                    assert.deepEqual(tile00Actual.sort(orderById), tile00Expected);
-                                    assert.deepEqual(tile10Actual.sort(orderById), tile10Expected);
-                                    assert.deepEqual(tile01Actual.sort(orderById), tile01Expected);
-                                    assert.deepEqual(tile11Actual.sort(orderById), tile11Expected);
 
+                                    const allFeatures = [... tile00Actual, ...tile10Actual,
+                                                         ...tile01Actual, ...tile11Actual];
+                                    for (let i = 0; i < allFeatures.length; i++) {
+                                        for (let j = i + 1; j < allFeatures.length; j++) {
+                                            const c1 = allFeatures[i];
+                                            const c2 = allFeatures[j];
+                                            if (c1.cartodb_id === c2.cartodb_id) {
+                                                assert.equal(c1._cdb_feature_count, c2._cdb_feature_count);
+                                            }
+                                        }
+                                    }
                                     done();
                                 });
                             });