From 506e16fc879aeba2de898c483934a3944271751e Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 18 Dec 2017 20:18:37 +0100 Subject: [PATCH 1/6] Experimental full-sample aggregation --- .../models/aggregation/aggregation-query.js | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 69118021..03c99081 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -186,5 +186,30 @@ const aggregationQueryTemplates = { FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) - ` + `, + + 'full-sample': ctx => ` + WITH _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ), _cdb_clusters AS ( + SELECT + MIN(cartodb_id) AS cartodb_id + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params + WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + GROUP BY + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) + ${dimensionNames(ctx)} + ) SELECT + _cdb_query.* + ${aggregateColumnNames(ctx)} + FROM + _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query + ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) +` + }; From bcd2fd8f88fee6997882400a39b2f80c4a81c721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 12:59:33 +0100 Subject: [PATCH 2/6] Export supported placements --- .../models/aggregation/aggregation-query.js | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 06697732..b98d0dbb 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -142,26 +142,26 @@ const aggregationQueryTemplates = { `, 'point-grid': ctx => ` - WITH _cdb_params AS ( + WITH _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ), + _cdb_clusters AS ( + SELECT + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx, + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params + WHERE the_geom_webmercator && _cdb_params.bbox + GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)} + ) SELECT - ${gridResolution(ctx)} AS res, - !bbox! AS bbox - ), - _cdb_clusters AS ( - SELECT - Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx, - Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy - ${dimensionDefs(ctx)} - ${aggregateColumnDefs(ctx)} - FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params - WHERE the_geom_webmercator && _cdb_params.bbox - GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)} - ) - SELECT - ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator - ${dimensionNames(ctx)} - ${aggregateColumnNames(ctx)} - FROM _cdb_clusters, _cdb_params + ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator + ${dimensionNames(ctx)} + ${aggregateColumnNames(ctx)} + FROM _cdb_clusters, _cdb_params `, 'point-sample': ctx => ` @@ -212,6 +212,7 @@ const aggregationQueryTemplates = { FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) -` - + ` }; + +module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates); From 5a3dd6a91467b9e8e3c3ae465a0821dabdd193d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 13:00:18 +0100 Subject: [PATCH 3/6] Use supported placemets of aggregation-query --- .../models/aggregation/aggregation-mapconfig.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index c6dbcad4..f5d40c0c 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -10,15 +10,13 @@ const { module.exports = class AggregationMapConfig extends MapConfig { static get PLACEMENTS () { - return [ - 'centroid', - 'point-grid', - 'point-sample' - ]; + return aggregationQuery.SUPPORTED_PLACEMENTS; } static get PLACEMENT () { - return AggregationMapConfig.PLACEMENTS[0]; + return AggregationMapConfig.PLACEMENTS[ + AggregationMapConfig.PLACEMENTS.indexOf('centroid') + ]; } static get THRESHOLD () { From d2828ecaff1d985fd915f467527f7744ed6ef0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 13:07:57 +0100 Subject: [PATCH 4/6] Update test --- test/acceptance/aggregation.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 73284582..4c2d790e 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -762,10 +762,10 @@ describe('aggregation', function () { } assert.deepEqual(body, { - errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample'], + errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample'], errors_with_context:[{ type: 'layer', - message: 'Invalid placement. Valid values: centroid, point-grid, point-sample', + message: 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample', layer: { id: "layer0", index: 0, From 4946ca688c0c86912d43fc67f09b2f0660e2787c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 16:17:13 +0100 Subject: [PATCH 5/6] Add test to check full-sample query --- test/acceptance/aggregation.js | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 4c2d790e..517a5692 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -692,6 +692,51 @@ describe('aggregation', function () { }); }); + it('aggregates with full-sample placement', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + resolution: 256, + aggregation: { + placement: 'point-grid', + columns: { + total: { + aggregate_function: 'sum', + aggregated_column: 'value' + }, + v_avg: { + aggregate_function: 'avg', + aggregated_column: 'value' + } + }, + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(0, 0, 0, { format: 'mvt' }, function (err, res, mvt) { + if (err) { + return done(err); + } + + const geojsonTile = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(geojsonTile.features)); + assert.ok(geojsonTile.features.length > 0); + + const feature = geojsonTile.features[0]; + + assert.ok(feature.properties.hasOwnProperty('value'), 'Missing value property'); + + done(); + }); + }); + it('should fail with bad resolution', function (done) { this.mapConfig = createVectorMapConfig([ { From cc9b190e5d6eb04e8970a3acc57fefd2de6425b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 16:17:37 +0100 Subject: [PATCH 6/6] Minor style formats --- .../models/aggregation/aggregation-query.js | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index b98d0dbb..fd860318 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -118,7 +118,8 @@ const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::dou const aggregationQueryTemplates = { 'centroid': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox @@ -142,7 +143,8 @@ const aggregationQueryTemplates = { `, 'point-grid': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox @@ -165,11 +167,13 @@ const aggregationQueryTemplates = { `, 'point-sample': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox - ), _cdb_clusters AS ( + ), + _cdb_clusters AS ( SELECT MIN(cartodb_id) AS cartodb_id ${dimensionDefs(ctx)} @@ -180,7 +184,8 @@ const aggregationQueryTemplates = { Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} - ) SELECT + ) + SELECT _cdb_clusters.cartodb_id, the_geom, the_geom_webmercator ${dimensionNames(ctx)} @@ -191,11 +196,13 @@ const aggregationQueryTemplates = { `, 'full-sample': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox - ), _cdb_clusters AS ( + ), + _cdb_clusters AS ( SELECT MIN(cartodb_id) AS cartodb_id ${dimensionDefs(ctx)}