cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1111 from Algunenano/cartodbless

Browse Source 
Render MVTs and aggregations without cartodb-postgresql
This commit is contained in:
Raúl Marín 2019-07-16 16:06:40 +02:00 committed by GitHub
commit ebf373e680
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 251 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/cartodb/backends/cluster.js
View File

@ -5,6 +5,9 @@ const dbParamsFromReqParams = require('../utils/database-params');
const debug = require('debug')('backend:cluster'); const debug = require('debug')('backend:cluster');
const AggregationMapConfig = require('../models/aggregation/aggregation-mapconfig'); const AggregationMapConfig = require('../models/aggregation/aggregation-mapconfig');
const windshaftUtils = require('windshaft').utils;
const webmercator = new windshaftUtils.WebMercatorHelper();
module.exports = class ClusterBackend { module.exports = class ClusterBackend {
getClusterFeatures (mapConfigProvider, params, callback) { getClusterFeatures (mapConfigProvider, params, callback) {
mapConfigProvider.getMapConfig((err, _mapConfig) => { mapConfigProvider.getMapConfig((err, _mapConfig) => {
@ -127,7 +130,7 @@ function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, ag
} , true); // use read-only transaction } , true); // use read-only transaction
} }
const schemaQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`; const schemaQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_cluster_schema LIMIT 0`;
const clusterFeaturesQuery = ctx => ` const clusterFeaturesQuery = ctx => `
WITH WITH
_cdb_params AS ( _cdb_params AS (
@ -156,9 +159,8 @@ const clusterFeaturesQuery = ctx => `
`; `;
const gridResolution = ctx => { const gridResolution = ctx => {
const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38); const zoomResolution = webmercator.getResolution({ z : Math.min(38, ctx.zoom) });
const pixelSize = `CDB_XYZ_Resolution(${ctx.zoom})`; return `${256/ctx.res} * (${zoomResolution})::double precision`;
return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
}; };
const aggregationQuery = ctx => ` const aggregationQuery = ctx => `

2
lib/cartodb/models/aggregation/aggregation-mapconfig.js
View File

@ -201,7 +201,7 @@ module.exports = class AggregationMapConfig extends MapConfig {
getLayerColumns (index, skipGeoms, callback) { getLayerColumns (index, skipGeoms, callback) {
const geomColumns = ['the_geom', 'the_geom_webmercator']; const geomColumns = ['the_geom', 'the_geom_webmercator'];
const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`; const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_aggregation_schema LIMIT 0`;
const layer = this.getLayer(index); const layer = this.getLayer(index);
this.pgConnection.getConnection(this.user, (err, connection) => { this.pgConnection.getConnection(this.user, (err, connection) => {

331
lib/cartodb/models/aggregation/aggregation-query.js
View File

@ -3,31 +3,8 @@
const timeDimension = require('./time-dimension'); const timeDimension = require('./time-dimension');
const DEFAULT_PLACEMENT = 'point-sample'; const DEFAULT_PLACEMENT = 'point-sample';
const windshaftUtils = require('windshaft').utils;
const webmercator = new windshaftUtils.WebMercatorHelper();
/**
* Returns a template function (function that accepts template parameters and returns a string)
* to generate an aggregation query.
* Valid options to define the query template are:
* - placement
* - columns
* - dimensions*
* The query template parameters taken by the result template function are:
* - sourceQuery
* - res
* - columns
* - dimensions
*/
const templateForOptions = (options) => {
let templateFn = defaultAggregationQueryTemplate;
if (!options.isDefaultAggregation) {
templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT];
if (!templateFn) {
throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
}
}
return templateFn;
};
function optionsToParams (options) { function optionsToParams (options) {
return { return {
@ -35,7 +12,9 @@ function optionsToParams (options) {
res: 256/options.resolution, res: 256/options.resolution,
columns: options.columns, columns: options.columns,
dimensions: options.dimensions, dimensions: options.dimensions,
filters: options.filters filters: options.filters,
placement: options.placement || DEFAULT_PLACEMENT,
isDefaultAggregation: options.isDefaultAggregation
}; };
} }
@ -53,7 +32,7 @@ function optionsToParams (options) {
* When placement, columns or dimensions are specified, columns are aggregated as requested * When placement, columns or dimensions are specified, columns are aggregated as requested
* (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement. * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
*/ */
const queryForOptions = (options) => templateForOptions(options)(optionsToParams(options)); const queryForOptions = (options) => aggregationQueryTemplate(optionsToParams(options));
module.exports = queryForOptions; module.exports = queryForOptions;
@ -87,7 +66,7 @@ const SUPPORTED_AGGREGATE_FUNCTIONS = {
sql: (column_name, params) => `max(${params.aggregated_column || column_name})` sql: (column_name, params) => `max(${params.aggregated_column || column_name})`
}, },
'mode': { 'mode': {
sql: (column_name, params) => `_cdb_mode(${params.aggregated_column || column_name})` sql: (column_name, params) => `mode() WITHIN GROUP (ORDER BY ${params.aggregated_column || column_name})`
} }
}; };
@ -106,16 +85,6 @@ const aggregateColumns = ctx => {
}, ctx.columns || {}); }, ctx.columns || {});
}; };
const aggregateColumnNames = (ctx, table) => {
let columns = aggregateColumns(ctx);
if (table) {
return sep(Object.keys(columns).map(
column_name => `${table}.${column_name}`
));
}
return sep(Object.keys(columns));
};
const aggregateExpression = (column_name, column_parameters) => { const aggregateExpression = (column_name, column_parameters) => {
const aggregate_function = column_parameters.aggregate_function || 'count'; const aggregate_function = column_parameters.aggregate_function || 'count';
const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
@ -297,156 +266,158 @@ const havingClause = ctx => {
// (i.e. each tile is divided into ctx.res*ctx.res cells). // (i.e. each tile is divided into ctx.res*ctx.res cells).
// We limit the the minimum resolution to avoid division by zero problems. The limit used is // We limit the the minimum resolution to avoid division by zero problems. The limit used is
// the pixel size of zoom level 30 (i.e. 1/2*(30+8) of the full earth web-mercator extent), which is about 0.15 mm. // the pixel size of zoom level 30 (i.e. 1/2*(30+8) of the full earth web-mercator extent), which is about 0.15 mm.
// Computing this using !scale_denominator!, !pixel_width! or !pixel_height! produces //
// inaccurate results due to rounding present in those values. // NOTE: We'd rather use !pixel_width!, but in Mapnik this value is extent / 256 for raster
// and extent / tile_extent {4096 default} for MVT, so since aggregations are always based
// on 256 we can't have the same query in both cases
// As this scale change doesn't happen in !scale_denominator! we use that instead
// NOTE 2: The 0.00028 is used in Mapnik (and replicated in pg-mvt) and comes from
// OGC's Styled Layer Descriptor Implementation Specification
const gridResolution = ctx => { const gridResolution = ctx => {
const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38); const minimumResolution = webmercator.getResolution({ z : 38 });
const pixelSize = 'CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))'; return `${256/ctx.res} * GREATEST(!scale_denominator! * 0.00028, ${minimumResolution})::double precision`;
return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
}; };
// Each aggregation cell is defined by the cell coordinates Floor(x/res), Floor(y/res), // SQL query to extract the boundaries of the area to be aggregated and the grid resolution
// i.e. they include the West and South borders but not the East and North ones. // cdb_{x-y}{min_max} return the limits of the tile. Aggregations do [min, max) in both axis
// So, to avoid picking points that don't belong to cells in the tile, given the tile // cdb_res: Aggregation resolution (as specified by gridResolution)
// limits Xmin, Ymin, Xmax, Ymax (bbox), we should select points that satisfy // cdb_point_bbox: Tile bounding box [min, max]
// Xmin <= x < Xmax and Ymin <= y < Ymax (with x, y from the_geom_webmercator) const gridInfoQuery = ctx => {
// On the other hand we can efficiently filter spatially (relying on spatial indexing) return `
// with `the_geom_webmercator && bbox` which is equivalent to SELECT
// Xmin <= x <= Xmax and Ymin <= y <= Ymax cdb_xmin,
// So, in order to be both efficient and accurate we will need to use both cdb_ymin,
// conditions for spatial filtering. cdb_xmax,
const spatialFilter = ` cdb_ymax,
(_cdb_query.the_geom_webmercator && _cdb_params.bbox) AND cdb_res,
ST_X(_cdb_query.the_geom_webmercator) >= _cdb_params.xmin AND ST_MakeEnvelope(cdb_xmin, cdb_ymin, cdb_xmax, cdb_ymax, 3857) AS cdb_point_bbox
ST_X(_cdb_query.the_geom_webmercator) < _cdb_params.xmax AND FROM
ST_Y(_cdb_query.the_geom_webmercator) >= _cdb_params.ymin AND (
ST_Y(_cdb_query.the_geom_webmercator) < _cdb_params.ymax SELECT
cdb_res,
CEIL (ST_XMIN(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_xmin,
FLOOR(ST_XMAX(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_xmax,
CEIL (ST_YMIN(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_ymin,
FLOOR(ST_YMAX(cdb_full_bbox) / cdb_res) * cdb_res AS cdb_ymax
FROM
(
SELECT
${gridResolution(ctx)} AS cdb_res,
!bbox! cdb_full_bbox
) _cdb_input_resources
) _cdb_grid_bbox_margins
`;
};
// Function to generate the resulting point for a cell from the aggregated data
const aggregatedPointWebMercator = (ctx) => {
switch (ctx.placement) {
// For centroid, we return the average of the cell
case 'centroid':
return ', ST_SetSRID(ST_MakePoint(AVG(cdb_x), AVG(cdb_y)), 3857) AS the_geom_webmercator';
// Middle point of the cell
case 'point-grid':
return `, ST_SetSRID(ST_MakePoint(cdb_pos_grid_x, cdb_pos_grid_y), 3857) AS the_geom_webmercator`;
// For point-sample we'll get a single point directly from the source
// If it's default aggregation we'll add the extra columns to keep backwards compatibility
case 'point-sample':
return '';
default:
throw new Error(`Invalid aggregation placement "${ctx.placement}"`);
}
};
// Function to generate the resulting point for a cell from the a join with the source
const aggregatedPointJoin = (ctx) => {
switch (ctx.placement) {
case 'centroid':
return '';
case 'point-grid':
return '';
// For point-sample we'll get a single point directly from the source
// If it's default aggregation we'll add the extra columns to keep backwards compatibility
case 'point-sample':
return `
NATURAL JOIN
(
SELECT ${ctx.isDefaultAggregation ? `*` : `cartodb_id, the_geom_webmercator`}
FROM
(
${ctx.sourceQuery}
) __cdb_src_query
) __cdb_query_columns
`; `;
// Notes: default:
// * We need to filter spatially using !bbox! to make the queries efficient because throw new Error('Invalid aggregation placement "${ctx.placement}"');
// the filter added by Mapnik (wrapping the query) }
// is only applied after the aggregation. };
// * This queries are used for rendering and the_geom is omitted in the results for better performance
// * If the MVT extent or tile buffer was 0 or a multiple of the resolution we could use directly // Function to generate the values common to all points in a cell
// the bbox for them, but in general we need to find the nearest cell limits inside the bbox. // By default we use the cell number (which is fast), but for point-grid we
// * bbox coordinates can have an error in the last digits; we apply a small correction before // get the coordinates of the mid point so we don't need to calculate them later
// applying CEIL or FLOOR to compensate for this, so that coordinates closer than a small (`eps`) // which requires extra data in the group by clause
// fraction of the cell size to a cell limit are moved to the exact limit. const aggregatedPosCoordinate = (ctx, coordinate) => {
const sqlParams = (ctx) => ` switch (ctx.placement) {
_cdb_res AS ( // For point-grid we return the coordinate of the middle point of the grid
SELECT case `point-grid`:
${gridResolution(ctx)} AS res, return `(FLOOR(cdb_${coordinate} / __cdb_grid_params.cdb_res) + 0.5) * __cdb_grid_params.cdb_res`;
!bbox! AS bbox,
(1E-6::double precision) AS eps // For other, we return the cell position (relative to the world)
), default:
_cdb_params AS ( return `FLOOR(cdb_${coordinate} / __cdb_grid_params.cdb_res)`;
SELECT }
res, };
bbox,
CEIL((ST_XMIN(bbox) - eps*res)/res)*res AS xmin,
FLOOR((ST_XMAX(bbox) + eps*res)/res)*res AS xmax, const aggregationQueryTemplate = ctx => `
CEIL((ST_YMIN(bbox) - eps*res)/res)*res AS ymin, WITH __cdb_grid_params AS
FLOOR((ST_YMAX(bbox) + eps*res)/res)*res AS ymax (
FROM _cdb_res ${gridInfoQuery(ctx)}
) )
SELECT * FROM
(
SELECT
min(cartodb_id) as cartodb_id
${aggregatedPointWebMercator(ctx)}
${dimensionDefs(ctx)}
${aggregateColumnDefs(ctx)}
FROM
(
SELECT
*,
${aggregatedPosCoordinate(ctx, 'x')} as cdb_pos_grid_x,
${aggregatedPosCoordinate(ctx, 'y')} as cdb_pos_grid_y
FROM
(
SELECT
__cdb_src_query.*,
ST_X(the_geom_webmercator) cdb_x,
ST_Y(the_geom_webmercator) cdb_y
FROM
(
${ctx.sourceQuery}
) __cdb_src_query, __cdb_grid_params
WHERE the_geom_webmercator && cdb_point_bbox
OFFSET 0
) __cdb_src_get_x_y, __cdb_grid_params
WHERE cdb_x < __cdb_grid_params.cdb_xmax AND cdb_y < __cdb_grid_params.cdb_ymax
) __cdb_src_gridded
GROUP BY cdb_pos_grid_x, cdb_pos_grid_y ${dimensionNames(ctx)}
${havingClause(ctx)}
) __cdb_aggregation_src
${aggregatedPointJoin(ctx)}
`; `;
// The special default aggregation includes all the columns of a sample row per grid cell and module.exports.SUPPORTED_PLACEMENTS = ['centroid', 'point-grid', 'point-sample'];
// the count (_cdb_feature_count) of the aggregated rows.
const defaultAggregationQueryTemplate = ctx => `
WITH ${sqlParams(ctx)},
_cdb_clusters AS (
SELECT
MIN(cartodb_id) AS cartodb_id
${dimensionDefs(ctx)}
${aggregateColumnDefs(ctx)}
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE ${spatialFilter}
GROUP BY
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
) SELECT
_cdb_query.*
${aggregateColumnNames(ctx)}
FROM
_cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
`;
const aggregationQueryTemplates = {
'centroid': ctx => `
WITH ${sqlParams(ctx)}
SELECT
MIN(_cdb_query.cartodb_id) AS cartodb_id,
ST_SetSRID(
ST_MakePoint(
AVG(ST_X(_cdb_query.the_geom_webmercator)),
AVG(ST_Y(_cdb_query.the_geom_webmercator))
), 3857
) AS the_geom_webmercator
${dimensionDefs(ctx)}
${aggregateColumnDefs(ctx)}
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE ${spatialFilter}
GROUP BY
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
${havingClause(ctx)}
`,
'point-grid': ctx => `
WITH ${sqlParams(ctx)},
_cdb_clusters AS (
SELECT
MIN(_cdb_query.cartodb_id) AS cartodb_id,
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy
${dimensionDefs(ctx)}
${aggregateColumnDefs(ctx)}
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE ${spatialFilter}
GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
${havingClause(ctx)}
)
SELECT
_cdb_clusters.cartodb_id AS cartodb_id,
ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator
${dimensionNames(ctx)}
${aggregateColumnNames(ctx)}
FROM _cdb_clusters, _cdb_params
`,
'point-sample': ctx => `
WITH ${sqlParams(ctx)},
_cdb_clusters AS (
SELECT
MIN(cartodb_id) AS cartodb_id
${dimensionDefs(ctx)}
${aggregateColumnDefs(ctx)}
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE ${spatialFilter}
GROUP BY
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
${havingClause(ctx)}
)
SELECT
_cdb_clusters.cartodb_id,
the_geom_webmercator
${dimensionNames(ctx, '_cdb_clusters')}
${aggregateColumnNames(ctx, '_cdb_clusters')}
FROM
_cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
`
};
module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates);
module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator'; module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator';
const clusterFeaturesQuery = ctx => ` const clusterFeaturesQuery = ctx => `

5
lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js
View File

@ -2,6 +2,7 @@
var queue = require('queue-async'); var queue = require('queue-async');
var _ = require('underscore'); var _ = require('underscore');
const AggregationMapConfig = require('../../aggregation/aggregation-mapconfig');
function MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend) { function MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend) {
this.overviewsMetadataBackend = overviewsMetadataBackend; this.overviewsMetadataBackend = overviewsMetadataBackend;
@ -14,7 +15,9 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function (user, requestMapCon
var layers = requestMapConfig.layers; var layers = requestMapConfig.layers;
var analysesResults = context.analysesResults; var analysesResults = context.analysesResults;
if (!layers || layers.length === 0) { const aggMapConfig = new AggregationMapConfig(null, requestMapConfig);
if (aggMapConfig.isVectorOnlyMapConfig() || aggMapConfig.isAggregationMapConfig() ||
!layers || layers.length === 0) {
return callback(null, requestMapConfig); return callback(null, requestMapConfig);
} }

99
lib/cartodb/utils/query-utils.js
View File

@ -1,48 +1,41 @@
'use strict'; 'use strict';
const SubstitutionTokens = require('./substitution-tokens'); const windshaftUtils = require('windshaft').utils;
function prepareQuery(sql) { module.exports.extractTableNames = function (query) {
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
}
module.exports.extractTableNames = function extractTableNames(query) {
return [ return [
'SELECT * FROM CDB_QueryTablesText($windshaft$', 'SELECT * FROM CDB_QueryTablesText($windshaft$',
prepareQuery(query), substituteDummyTokens(query),
'$windshaft$) as tablenames' '$windshaft$) as tablenames'
].join(''); ].join('');
}; };
module.exports.getQueryActualRowCount = function (query) { module.exports.getQueryActualRowCount = function (query) {
return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`; return `select COUNT(*) AS rows FROM (${substituteDummyTokens(query)}) AS __cdb_query`;
}; };
function getQueryRowEstimation(query) { function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows'; return 'select CDB_EstimateRowCount($windshaft$' + substituteDummyTokens(query) + '$windshaft$) as rows';
} }
module.exports.getQueryRowEstimation = getQueryRowEstimation; module.exports.getQueryRowEstimation = getQueryRowEstimation;
function getQueryGeometryType(query, geometryColumn) {
return `
SELECT ST_GeometryType(${geometryColumn}) AS geom_type
FROM (${substituteDummyTokens(query)}) AS __cdb_query
WHERE ${geometryColumn} IS NOT NULL
LIMIT 1
`;
}
module.exports.getQueryGeometryType = getQueryGeometryType;
module.exports.getAggregationMetadata = ctx => ` module.exports.getAggregationMetadata = ctx => `
WITH WITH
rowEstimation AS ( rowEstimation AS (
${getQueryRowEstimation(ctx.query)} ${getQueryRowEstimation(ctx.query)}
), ),
geometryType AS ( geometryType AS (
SELECT ST_GeometryType(${ctx.geometryColumn}) as geom_type ${getQueryGeometryType(ctx.query, ctx.geometryColumn)}
FROM (${ctx.query}) AS __cdb_query WHERE ${ctx.geometryColumn} IS NOT NULL LIMIT 1
) )
SELECT SELECT
rows AS count, rows AS count,
@ -85,7 +78,7 @@ module.exports.getQueryTopCategories = function (query, column, topN, includeNul
const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`; const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`;
return ` return `
SELECT ${column} AS category, COUNT(*) AS frequency SELECT ${column} AS category, COUNT(*) AS frequency
FROM (${query}) AS __cdb_query FROM (${substituteDummyTokens(query)}) AS __cdb_query
${where} ${where}
GROUP BY ${column} ORDER BY 2 DESC GROUP BY ${column} ORDER BY 2 DESC
LIMIT ${topN} LIMIT ${topN}
@ -116,7 +109,7 @@ module.exports.getQuerySample = function (query, sampleProb, limit = null, rando
SELECT setseed(${randomSeed}) SELECT setseed(${randomSeed})
) )
SELECT ${columnSelector(columns)} SELECT ${columnSelector(columns)}
FROM (${query}) AS __cdb_query FROM (${substituteDummyTokens(query)}) AS __cdb_query
WHERE random() < ${sampleProb} WHERE random() < ${sampleProb}
${limitClause} ${limitClause}
`; `;
@ -157,19 +150,11 @@ function simpleQueryTable(sql) {
return false; return false;
} }
module.exports.getQueryGeometryType = function (query, geometryColumn) {
return `
SELECT ST_GeometryType(${geometryColumn}) AS geom_type
FROM (${query}) AS __cdb_query
WHERE ${geometryColumn} IS NOT NULL
LIMIT 1
`;
};
function getQueryLimited(query, limit = 0) { function getQueryLimited(query, limit = 0) {
return ` return `
SELECT * SELECT *
FROM (${query}) AS __cdb_query FROM (${substituteDummyTokens(query)}) AS __cdb_query
LIMIT ${limit} LIMIT ${limit}
`; `;
} }
@ -181,32 +166,32 @@ function queryPromise(dbConnection, query) {
} }
function substituteDummyTokens(sql) { function substituteDummyTokens(sql) {
return sql && SubstitutionTokens.replace(sql, { return subsituteTokensForZoom(sql, 0);
bbox: 'ST_MakeEnvelope(0,0,0,0)',
scale_denominator: '0',
pixel_width: '1',
pixel_height: '1'
});
} }
function subsituteTokensForZoom(sql, zoom, singleTile=false) { function subsituteTokensForZoom(sql, zoom) {
const tileRes = 256; if (!sql) {
const wmSize = 6378137.0*2*Math.PI; return undefined;
const nTiles = Math.pow(2, zoom);
const tileSize = wmSize / nTiles;
const resolution = tileSize / tileRes;
const scaleDenominator = resolution / 0.00028;
const x0 = -wmSize/2, y0 = -wmSize/2;
let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`;
if (singleTile) {
bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`;
} }
return SubstitutionTokens.replace(sql, { const affectedTableRegexCache = {
bbox: bbox, bbox: /!bbox!/g,
scale_denominator: scaleDenominator, scale_denominator: /!scale_denominator!/g,
pixel_width: resolution, pixel_width: /!pixel_width!/g,
pixel_height: resolution pixel_height: /!pixel_height!/g
}); };
const webmercator = new windshaftUtils.WebMercatorHelper();
const resolution = webmercator.getResolution({ z : zoom });
const scaleDenominator = resolution.dividedBy(0.00028);
// We always use the whole world as the bbox
const extent = webmercator.getExtent({ x : 0, y : 0, z : 0 });
return sql
.replace(affectedTableRegexCache.bbox,
`ST_MakeEnvelope(${extent.xmin}, ${extent.ymin}, ${extent.xmax}, ${extent.ymax}, 3857)`)
.replace(affectedTableRegexCache.scale_denominator, scaleDenominator)
.replace(affectedTableRegexCache.pixel_width, resolution)
.replace(affectedTableRegexCache.pixel_height, resolution);
} }
module.exports.queryPromise = queryPromise; module.exports.queryPromise = queryPromise;

142
test/acceptance/aggregation.js
View File

@ -6,6 +6,9 @@ const assert = require('../support/assert');
const TestClient = require('../support/test-client'); const TestClient = require('../support/test-client');
const serverOptions = require('../../lib/cartodb/server_options'); const serverOptions = require('../../lib/cartodb/server_options');
const windshaftUtils = require('windshaft').utils;
const webmercator = new windshaftUtils.WebMercatorHelper();
const suites = [ const suites = [
{ {
desc: 'mvt (mapnik)', desc: 'mvt (mapnik)',
@ -96,9 +99,9 @@ describe('aggregation', function () {
const POLYGONS_SQL_1 = ` const POLYGONS_SQL_1 = `
select select
x + 4 as cartodb_id, x + 4 as cartodb_id,
st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 100000)::geometry as the_geom, st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326), 10) as the_geom,
st_transform( st_transform(
st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 100000)::geometry, st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326), 10),
3857 3857
) as the_geom_webmercator, ) as the_geom_webmercator,
x as value x as value
@ -109,15 +112,15 @@ describe('aggregation', function () {
WITH hgrid AS ( WITH hgrid AS (
SELECT SELECT
CDB_RectangleGrid ( CDB_RectangleGrid (
ST_Expand(!bbox!, CDB_XYZ_Resolution(1) * 12), ST_Expand(!bbox!, ${webmercator.getResolution({ z : 1 })} * 12),
CDB_XYZ_Resolution(1) * 12, ${webmercator.getResolution({ z : 1 })} * 12,
CDB_XYZ_Resolution(1) * 12 ${webmercator.getResolution({ z : 1 })} * 12
) as cell ) as cell
) )
SELECT SELECT
hgrid.cell as the_geom_webmercator, hgrid.cell as the_geom_webmercator,
count(1) as agg_value, count(1) as agg_value,
count(1) /power( 12 * CDB_XYZ_Resolution(1), 2 ) as agg_value_density, count(1) /power( 12 * ${webmercator.getResolution({ z : 1 })}, 2 ) as agg_value_density,
row_number() over () as cartodb_id row_number() over () as cartodb_id
FROM hgrid, (<%= sql %>) i FROM hgrid, (<%= sql %>) i
WHERE ST_Intersects(i.the_geom_webmercator, hgrid.cell) GROUP BY hgrid.cell WHERE ST_Intersects(i.the_geom_webmercator, hgrid.cell) GROUP BY hgrid.cell
@ -202,9 +205,9 @@ describe('aggregation', function () {
// | | | | | | | // | | | | | | |
// Tile 0, 1 -+---+---+---+- Tile 1,1 // Tile 0, 1 -+---+---+---+- Tile 1,1
// //
const POINTS_SQL_GRID = ` const POINTS_SQL_GRID = (z, resolution) => `
WITH params AS ( WITH params AS (
SELECT CDB_XYZ_Resolution($Z)*$resolution AS l -- cell size for Z, resolution SELECT ${webmercator.getResolution({ z : z })}*${resolution} AS l -- cell size for Z, resolution
) )
SELECT SELECT
row_number() OVER () AS cartodb_id, row_number() OVER () AS cartodb_id,
@ -224,8 +227,8 @@ describe('aggregation', function () {
const POINTS_SQL_CELL = ` const POINTS_SQL_CELL = `
SELECT SELECT
1 AS cartodb_id, 1 AS cartodb_id,
ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857) AS the_geom_webmercator, ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857) AS the_geom_webmercator,
ST_Transform(ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857), 4326) AS the_geom ST_Transform(ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857), 4326) AS the_geom
UNION ALL SELECT UNION ALL SELECT
2 AS cartodb_id, 2 AS cartodb_id,
ST_SetSRID(ST_MakePoint(18181005.9, -18181044.0), 3857) AS the_geom_webmercator, ST_SetSRID(ST_MakePoint(18181005.9, -18181044.0), 3857) AS the_geom_webmercator,
@ -236,8 +239,8 @@ describe('aggregation', function () {
ST_Transform(ST_SetSRID(ST_MakePoint(18181005.87, -18181043.94), 3857), 4326) AS the_geom ST_Transform(ST_SetSRID(ST_MakePoint(18181005.87, -18181043.94), 3857), 4326) AS the_geom
UNION ALL SELECT UNION ALL SELECT
4 AS cartodb_id, 4 AS cartodb_id,
ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857) AS the_geom_webmercator, ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857) AS the_geom_webmercator,
ST_Transform(ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857), 4326) AS the_geom ST_Transform(ST_SetSRID(ST_MakePoint(18181005.82, -18181043.9), 3857), 4326) AS the_geom
`; `;
// Points positioned inside one cell of Z=20, X=1000000, X=1000000 (inner cell not on border) // Points positioned inside one cell of Z=20, X=1000000, X=1000000 (inner cell not on border)
@ -249,8 +252,8 @@ describe('aggregation', function () {
ST_Transform(ST_SetSRID(ST_MakePoint(18181005.95, -18181043.8), 3857), 4326) AS the_geom ST_Transform(ST_SetSRID(ST_MakePoint(18181005.95, -18181043.8), 3857), 4326) AS the_geom
UNION ALL SELECT UNION ALL SELECT
2 AS cartodb_id, 2 AS cartodb_id,
ST_SetSRID(ST_MakePoint(18181006.09, -18181043.72), 3857) AS the_geom_webmercator, ST_SetSRID(ST_MakePoint(18181006.09, -18181043.74), 3857) AS the_geom_webmercator,
ST_Transform(ST_SetSRID(ST_MakePoint(18181006.09, -18181043.72), 3857), 4326) AS the_geom ST_Transform(ST_SetSRID(ST_MakePoint(18181006.09, -18181043.74), 3857), 4326) AS the_geom
UNION ALL SELECT UNION ALL SELECT
3 AS cartodb_id, 3 AS cartodb_id,
ST_SetSRID(ST_MakePoint(18181006.02, -18181043.79), 3857) AS the_geom_webmercator, ST_SetSRID(ST_MakePoint(18181006.02, -18181043.79), 3857) AS the_geom_webmercator,
@ -521,7 +524,7 @@ describe('aggregation', function () {
}); });
['centroid', 'point-sample', 'point-grid'].forEach(placement => { ['centroid', 'point-sample', 'point-grid'].forEach(placement => {
it('should provide all the requested columns in non-default aggregation ', it('should provide all the requested columns in non-default aggregation: ' + placement,
function (done) { function (done) {
const response = { const response = {
status: 200, status: 200,
@ -570,7 +573,7 @@ describe('aggregation', function () {
}); });
}); });
it('should provide only the requested columns in non-default aggregation ', it('should provide only the requested columns in non-default aggregation: ' + placement,
function (done) { function (done) {
this.mapConfig = createVectorMapConfig([ this.mapConfig = createVectorMapConfig([
{ {
@ -2351,7 +2354,7 @@ describe('aggregation', function () {
threshold: 1, threshold: 1,
columns: { columns: {
value: { value: {
aggregate_function: 'sum', aggregate_function: 'mode',
aggregated_column: 'value' aggregated_column: 'value'
} }
}, },
@ -2397,7 +2400,7 @@ describe('aggregation', function () {
threshold: 1, threshold: 1,
columns: { columns: {
value: { value: {
aggregate_function: 'sum', aggregate_function: 'mode',
aggregated_column: 'value' aggregated_column: 'value'
} }
}, },
@ -2970,7 +2973,7 @@ describe('aggregation', function () {
it(`for ${placement} each aggr. cell is in a single tile`, function (done) { it(`for ${placement} each aggr. cell is in a single tile`, function (done) {
const z = 1; const z = 1;
const resolution = 1; const resolution = 1;
const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution); const query = POINTS_SQL_GRID(z, resolution);
this.mapConfig = { this.mapConfig = {
version: '1.6.0', version: '1.6.0',
buffersize: { 'mvt': 0 }, buffersize: { 'mvt': 0 },
@ -3015,32 +3018,17 @@ describe('aggregation', function () {
} }
const tile11 = JSON.parse(mvt.toGeoJSONSync(0)); const tile11 = JSON.parse(mvt.toGeoJSONSync(0));
const tile00Expected = [ // There needs to be 13 points
{ cartodb_id: 4, _cdb_feature_count: 2 }, const count_features = ((tile) =>
{ cartodb_id: 7, _cdb_feature_count: 1 } tile.features.map(f => f.properties)
]; .map(f => f._cdb_feature_count)
const tile10Expected = [ .reduce((a,b) => a + b, 0));
{ cartodb_id: 5, _cdb_feature_count: 2 },
{ cartodb_id: 6, _cdb_feature_count: 1 }, const tile00Count = count_features(tile00);
{ cartodb_id: 8, _cdb_feature_count: 1 }, const tile10Count = count_features(tile10);
{ cartodb_id: 9, _cdb_feature_count: 1 } const tile01Count = count_features(tile01);
]; const tile11Count = count_features(tile11);
const tile01Expected = [ assert.equal(13, tile00Count + tile10Count + tile01Count + tile11Count);
{ cartodb_id: 1, _cdb_feature_count: 2 }
];
const tile11Expected = [
{ cartodb_id: 2, _cdb_feature_count: 2 },
{ cartodb_id: 3, _cdb_feature_count: 1 }
];
const tile00Actual = tile00.features.map(f => f.properties);
const tile10Actual = tile10.features.map(f => f.properties);
const tile01Actual = tile01.features.map(f => f.properties);
const tile11Actual = tile11.features.map(f => f.properties);
const orderById = (a, b) => a.cartodb_id - b.cartodb_id;
assert.deepEqual(tile00Actual.sort(orderById), tile00Expected);
assert.deepEqual(tile10Actual.sort(orderById), tile10Expected);
assert.deepEqual(tile01Actual.sort(orderById), tile01Expected);
assert.deepEqual(tile11Actual.sort(orderById), tile11Expected);
done(); done();
}); });
@ -3056,7 +3044,7 @@ describe('aggregation', function () {
const z = 1; const z = 1;
const resolution = 2; const resolution = 2;
// space the test points by half the resolution: // space the test points by half the resolution:
const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution/2); const query = POINTS_SQL_GRID(z, resolution / 2);
this.mapConfig = { this.mapConfig = {
version: '1.6.0', version: '1.6.0',
@ -3133,20 +3121,11 @@ describe('aggregation', function () {
}); });
it(`for ${placement} includes complete cells in buffer`, function (done) { it(`for ${placement} includes complete cells in buffer`, function (done) {
if (!usePostGIS && placement !== 'point-grid') {
// Mapnik seem to filter query results by its (inaccurate) bbox,
// which makes some aggregated clusters get lost here.
// The point-grid placement is resilient to this problem because the result
// coordinates are moved to cluster cell centers, so they're well within
// bbox limits.
this.testClient = new TestClient({});
return done();
}
// use buffersize coincident with resolution, the buffer should include neighbour cells // use buffersize coincident with resolution, the buffer should include neighbour cells
const z = 2; const z = 2;
const resolution = 1; const resolution = 1;
const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution); const query = POINTS_SQL_GRID(z, resolution);
this.mapConfig = { this.mapConfig = {
version: '1.6.0', version: '1.6.0',
@ -3192,49 +3171,24 @@ describe('aggregation', function () {
} }
const tile11 = JSON.parse(mvt.toGeoJSONSync(0)); const tile11 = JSON.parse(mvt.toGeoJSONSync(0));
const tile00Expected = [ // We check that if an id/cell is present in multiple tiles,
{ _cdb_feature_count: 2, cartodb_id: 1 }, // it always contains the same amount of features
{ _cdb_feature_count: 2, cartodb_id: 2 },
{ _cdb_feature_count: 2, cartodb_id: 4 },
{ _cdb_feature_count: 2, cartodb_id: 5 },
{ _cdb_feature_count: 1, cartodb_id: 7 },
{ _cdb_feature_count: 1, cartodb_id: 8 }
];
const tile10Expected = [
{ _cdb_feature_count: 2, cartodb_id: 1 },
{ _cdb_feature_count: 2, cartodb_id: 2 },
{ _cdb_feature_count: 1, cartodb_id: 3 },
{ _cdb_feature_count: 2, cartodb_id: 4 },
{ _cdb_feature_count: 2, cartodb_id: 5 },
{ _cdb_feature_count: 1, cartodb_id: 6 },
{ _cdb_feature_count: 1, cartodb_id: 7 },
{ _cdb_feature_count: 1, cartodb_id: 8 },
{ _cdb_feature_count: 1, cartodb_id: 9 }
];
const tile01Expected = [
{ _cdb_feature_count: 2, cartodb_id: 1 },
{ _cdb_feature_count: 2, cartodb_id: 2 },
{ _cdb_feature_count: 2, cartodb_id: 4 },
{ _cdb_feature_count: 2, cartodb_id: 5 }
];
const tile11Expected = [
{ _cdb_feature_count: 2, cartodb_id: 1 },
{ _cdb_feature_count: 2, cartodb_id: 2 },
{ _cdb_feature_count: 1, cartodb_id: 3 },
{ _cdb_feature_count: 2, cartodb_id: 4 },
{ _cdb_feature_count: 2, cartodb_id: 5 },
{ _cdb_feature_count: 1, cartodb_id: 6 }
];
const tile00Actual = tile00.features.map(f => f.properties); const tile00Actual = tile00.features.map(f => f.properties);
const tile10Actual = tile10.features.map(f => f.properties); const tile10Actual = tile10.features.map(f => f.properties);
const tile01Actual = tile01.features.map(f => f.properties); const tile01Actual = tile01.features.map(f => f.properties);
const tile11Actual = tile11.features.map(f => f.properties); const tile11Actual = tile11.features.map(f => f.properties);
const orderById = (a, b) => a.cartodb_id - b.cartodb_id;
assert.deepEqual(tile00Actual.sort(orderById), tile00Expected);
assert.deepEqual(tile10Actual.sort(orderById), tile10Expected);
assert.deepEqual(tile01Actual.sort(orderById), tile01Expected);
assert.deepEqual(tile11Actual.sort(orderById), tile11Expected);
const allFeatures = [... tile00Actual, ...tile10Actual,
...tile01Actual, ...tile11Actual];
for (let i = 0; i < allFeatures.length; i++) {
for (let j = i + 1; j < allFeatures.length; j++) {
const c1 = allFeatures[i];
const c2 = allFeatures[j];
if (c1.cartodb_id === c2.cartodb_id) {
assert.equal(c1._cdb_feature_count, c2._cdb_feature_count);
}
}
}
done(); done();
}); });
}); });