Merge branch 'full-sample' of github.com:CartoDB/Windshaft-cartodb into pg-mvt-do-not-filter-columns
This commit is contained in:
Daniel García Aubert
commit
062e6f9594
@ -16,10 +16,6 @@ module.exports = class AggregationMapConfig extends MapConfig {
|
||||
return aggregationQuery.SUPPORTED_PLACEMENTS;
|
||||
}
|
||||
|
||||
static get PLACEMENT () {
|
||||
return AggregationMapConfig.PLACEMENTS.find(placement => placement === 'centroid');
|
||||
}
|
||||
|
||||
static get THRESHOLD () {
|
||||
return 1e5; // 100K
|
||||
}
|
||||
@ -58,9 +54,11 @@ module.exports = class AggregationMapConfig extends MapConfig {
|
||||
getAggregatedQuery (index) {
|
||||
const { sql_raw, sql } = this.getLayer(index).options;
|
||||
const {
|
||||
// The default aggregation has no placement, columns or dimensions;
|
||||
// this enables the special "full-sample" aggregation.
|
||||
resolution = AggregationMapConfig.RESOLUTION,
|
||||
threshold = AggregationMapConfig.THRESHOLD,
|
||||
placement = AggregationMapConfig.PLACEMENT,
|
||||
placement = null,
|
||||
columns = {},
|
||||
dimensions = {}
|
||||
} = this.getAggregation(index);
|
||||
|
||||
@ -1,8 +1,12 @@
|
||||
const DEFAULT_PLACEMENT = 'point-sample';
|
||||
|
||||
/**
|
||||
* Returns a template function (function that accepts template parameters and returns a string)
|
||||
* to generate an aggregation query.
|
||||
* Valid options to define the query template are:
|
||||
* - placement
|
||||
* - columns
|
||||
* - dimensions*
|
||||
* The query template parameters taken by the result template function are:
|
||||
* - sourceQuery
|
||||
* - res
|
||||
@ -10,13 +14,28 @@
|
||||
* - dimensions
|
||||
*/
|
||||
const templateForOptions = (options) => {
|
||||
let templateFn = aggregationQueryTemplates[options.placement];
|
||||
if (!templateFn) {
|
||||
throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
|
||||
let templateFn = defaultAggregationQueryTemplate;
|
||||
if (!isDefaultAggregation(options)) {
|
||||
templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT];
|
||||
if (!templateFn) {
|
||||
throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
|
||||
}
|
||||
}
|
||||
return templateFn;
|
||||
};
|
||||
|
||||
const isEmptyParameter = (parameter) => !parameter || Object.keys(parameter).length === 0;
|
||||
|
||||
/**
|
||||
* When no placement, columns or dimensions are specified in the aggregation
|
||||
* a special default aggregation is used.
|
||||
*/
|
||||
const isDefaultAggregation = (options) => {
|
||||
return !options || (
|
||||
!options.placement && isEmptyParameter(options.columns) && isEmptyParameter(options.dimensions)
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Generates an aggregation query given the aggregation options:
|
||||
* - query
|
||||
@ -25,6 +44,11 @@ const templateForOptions = (options) => {
|
||||
* - columns
|
||||
* - placement
|
||||
* - dimensions
|
||||
*
|
||||
* The default aggregation (when no explicit placement, columns or dimensions are present) returns
|
||||
* a sample record (with all the original columns and _cdb_feature_count) for each aggregation group.
|
||||
* When placement, columns or dimensions are specified, columns are aggregated as requested
|
||||
* (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
|
||||
*/
|
||||
const queryForOptions = (options) => templateForOptions(options)({
|
||||
sourceQuery: options.query,
|
||||
@ -35,6 +59,9 @@ const queryForOptions = (options) => templateForOptions(options)({
|
||||
|
||||
module.exports = queryForOptions;
|
||||
|
||||
// Checks if the aggregation parameters represent the default (full-sample) aggregation
|
||||
module.exports.isDefaultAggregation = isDefaultAggregation;
|
||||
|
||||
const SUPPORTED_AGGREGATE_FUNCTIONS = {
|
||||
'count': {
|
||||
sql: (column_name, params) => `count(${params.aggregated_column || '*'})`
|
||||
@ -71,8 +98,13 @@ const aggregateColumns = ctx => {
|
||||
}, ctx.columns || {});
|
||||
};
|
||||
|
||||
const aggregateColumnNames = ctx => {
|
||||
const aggregateColumnNames = (ctx, table) => {
|
||||
let columns = aggregateColumns(ctx);
|
||||
if (table) {
|
||||
return sep(Object.keys(columns).map(
|
||||
column_name => `${table}.${column_name}`
|
||||
));
|
||||
}
|
||||
return sep(Object.keys(columns));
|
||||
};
|
||||
|
||||
@ -92,8 +124,14 @@ const aggregateColumnDefs = ctx => {
|
||||
|
||||
const aggregateDimensions = ctx => ctx.dimensions || {};
|
||||
|
||||
const dimensionNames = ctx => {
|
||||
return sep(Object.keys(aggregateDimensions(ctx)));
|
||||
const dimensionNames = (ctx, table) => {
|
||||
let dimensions = aggregateDimensions(ctx);
|
||||
if (table) {
|
||||
return sep(Object.keys(dimensions).map(
|
||||
dimension_name => `${table}.${dimension_name}`
|
||||
));
|
||||
}
|
||||
return sep(Object.keys(dimensions));
|
||||
};
|
||||
|
||||
const dimensionDefs = ctx => {
|
||||
@ -116,6 +154,34 @@ const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::dou
|
||||
// is only applied after the aggregation.
|
||||
// * This queries are used for rendering and the_geom is omitted in the results for better performance
|
||||
|
||||
// The special default aggregation includes all the columns of a sample row per grid cell and
|
||||
// the count (_cdb_feature_count) of the aggregated rows.
|
||||
const defaultAggregationQueryTemplate = ctx => `
|
||||
WITH
|
||||
_cdb_params AS (
|
||||
SELECT
|
||||
${gridResolution(ctx)} AS res,
|
||||
!bbox! AS bbox
|
||||
),
|
||||
_cdb_clusters AS (
|
||||
SELECT
|
||||
MIN(cartodb_id) AS cartodb_id
|
||||
${dimensionDefs(ctx)}
|
||||
${aggregateColumnDefs(ctx)}
|
||||
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
|
||||
WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
|
||||
GROUP BY
|
||||
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
|
||||
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
|
||||
${dimensionNames(ctx)}
|
||||
) SELECT
|
||||
_cdb_query.*
|
||||
${aggregateColumnNames(ctx)}
|
||||
FROM
|
||||
_cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
|
||||
ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
|
||||
`;
|
||||
|
||||
const aggregationQueryTemplates = {
|
||||
'centroid': ctx => `
|
||||
WITH
|
||||
@ -188,38 +254,13 @@ const aggregationQueryTemplates = {
|
||||
SELECT
|
||||
_cdb_clusters.cartodb_id,
|
||||
the_geom, the_geom_webmercator
|
||||
${dimensionNames(ctx)}
|
||||
${aggregateColumnNames(ctx)}
|
||||
FROM
|
||||
_cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
|
||||
ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
|
||||
`,
|
||||
|
||||
'full-sample': ctx => `
|
||||
WITH
|
||||
_cdb_params AS (
|
||||
SELECT
|
||||
${gridResolution(ctx)} AS res,
|
||||
!bbox! AS bbox
|
||||
),
|
||||
_cdb_clusters AS (
|
||||
SELECT
|
||||
MIN(cartodb_id) AS cartodb_id
|
||||
${dimensionDefs(ctx)}
|
||||
${aggregateColumnDefs(ctx)}
|
||||
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
|
||||
WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
|
||||
GROUP BY
|
||||
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
|
||||
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
|
||||
${dimensionNames(ctx)}
|
||||
) SELECT
|
||||
_cdb_query.*
|
||||
${aggregateColumnNames(ctx)}
|
||||
${dimensionNames(ctx, '_cdb_query')}
|
||||
${aggregateColumnNames(ctx, '_cdb_clusters')}
|
||||
FROM
|
||||
_cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
|
||||
ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
|
||||
`
|
||||
|
||||
};
|
||||
|
||||
module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates);
|
||||
|
||||
@ -289,7 +289,8 @@ describe('aggregation', function () {
|
||||
options: {
|
||||
sql: POINTS_SQL_2,
|
||||
aggregation: {
|
||||
threshold: 1
|
||||
threshold: 1,
|
||||
placement: 'centroid'
|
||||
},
|
||||
cartocss: '#layer { marker-width: [value]; }',
|
||||
cartocss_version: '2.3.0'
|
||||
@ -309,6 +310,45 @@ describe('aggregation', function () {
|
||||
});
|
||||
});
|
||||
|
||||
it('should provide all columns in the default aggregation ',
|
||||
function (done) {
|
||||
const response = {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8'
|
||||
}
|
||||
};
|
||||
|
||||
this.mapConfig = createVectorMapConfig([
|
||||
{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: POINTS_SQL_2,
|
||||
aggregation: {
|
||||
threshold: 1
|
||||
},
|
||||
cartocss: '#layer { marker-width: [value]; }',
|
||||
cartocss_version: '2.3.0'
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
this.testClient = new TestClient(this.mapConfig);
|
||||
this.testClient.getLayergroup({ response }, (err, body) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
|
||||
assert.equal(typeof body.metadata, 'object');
|
||||
assert.ok(Array.isArray(body.metadata.layers));
|
||||
|
||||
body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt));
|
||||
body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png));
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should skip aggregation to create a layergroup with aggregation defined already', function (done) {
|
||||
const mapConfig = createVectorMapConfig([
|
||||
{
|
||||
@ -569,6 +609,188 @@ describe('aggregation', function () {
|
||||
});
|
||||
});
|
||||
|
||||
['centroid', 'point-sample', 'point-grid'].forEach(placement => {
|
||||
it(`dimensions should work for ${placement} placement`, function(done) {
|
||||
|
||||
// FIXME: skip until pg-mvt renderer is able to return all columns
|
||||
if (process.env.POSTGIS_VERSION === '2.4') {
|
||||
return done();
|
||||
}
|
||||
|
||||
this.mapConfig = createVectorMapConfig([
|
||||
{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: POINTS_SQL_1,
|
||||
aggregation: {
|
||||
placement: placement ,
|
||||
threshold: 1,
|
||||
dimensions: {
|
||||
value: "value"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
this.testClient = new TestClient(this.mapConfig);
|
||||
const options = {
|
||||
format: 'mvt'
|
||||
};
|
||||
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const tileJSON = tile.toJSON();
|
||||
|
||||
tileJSON[0].features.forEach(
|
||||
feature => assert.equal(typeof feature.properties.value, 'number')
|
||||
);
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it(`dimensions should trigger non-default aggregation`, function(done) {
|
||||
|
||||
// FIXME: skip until pg-mvt renderer is able to return all columns
|
||||
if (process.env.POSTGIS_VERSION === '2.4') {
|
||||
return done();
|
||||
}
|
||||
|
||||
this.mapConfig = createVectorMapConfig([
|
||||
{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: POINTS_SQL_2,
|
||||
aggregation: {
|
||||
threshold: 1,
|
||||
dimensions: {
|
||||
value: "value"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
this.testClient = new TestClient(this.mapConfig);
|
||||
const options = {
|
||||
format: 'mvt'
|
||||
};
|
||||
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const tileJSON = tile.toJSON();
|
||||
|
||||
tileJSON[0].features.forEach(
|
||||
feature => assert.equal(typeof feature.properties.value, 'number')
|
||||
);
|
||||
tileJSON[0].features.forEach(
|
||||
feature => assert.equal(typeof feature.properties.sqrt_value, 'undefined')
|
||||
);
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it(`aggregation columns should trigger non-default aggregation`, function(done) {
|
||||
|
||||
// FIXME: skip until pg-mvt renderer is able to return all columns
|
||||
if (process.env.POSTGIS_VERSION === '2.4') {
|
||||
return done();
|
||||
}
|
||||
|
||||
this.mapConfig = createVectorMapConfig([
|
||||
{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: POINTS_SQL_2,
|
||||
aggregation: {
|
||||
threshold: 1,
|
||||
columns: {
|
||||
value: {
|
||||
aggregate_function: 'sum',
|
||||
aggregated_column: 'value'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
this.testClient = new TestClient(this.mapConfig);
|
||||
const options = {
|
||||
format: 'mvt'
|
||||
};
|
||||
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const tileJSON = tile.toJSON();
|
||||
|
||||
tileJSON[0].features.forEach(
|
||||
feature => assert.equal(typeof feature.properties.value, 'number')
|
||||
);
|
||||
tileJSON[0].features.forEach(
|
||||
feature => assert.equal(typeof feature.properties.sqrt_value, 'undefined')
|
||||
);
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
['centroid', 'point-sample', 'point-grid'].forEach(placement => {
|
||||
it(`aggregations with base column names should work for ${placement} placement`, function(done) {
|
||||
|
||||
// FIXME: skip until pg-mvt renderer is able to return all columns
|
||||
if (process.env.POSTGIS_VERSION === '2.4') {
|
||||
return done();
|
||||
}
|
||||
|
||||
this.mapConfig = createVectorMapConfig([
|
||||
{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: POINTS_SQL_1,
|
||||
aggregation: {
|
||||
placement: placement ,
|
||||
threshold: 1,
|
||||
columns: {
|
||||
value: {
|
||||
aggregate_function: 'sum',
|
||||
aggregated_column: 'value'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
this.testClient = new TestClient(this.mapConfig);
|
||||
const options = {
|
||||
format: 'mvt'
|
||||
};
|
||||
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const tileJSON = tile.toJSON();
|
||||
|
||||
tileJSON[0].features.forEach(
|
||||
feature => assert.equal(typeof feature.properties.value, 'number')
|
||||
);
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should work when the sql has single quotes', function (done) {
|
||||
this.mapConfig = createVectorMapConfig([
|
||||
{
|
||||
@ -576,6 +798,7 @@ describe('aggregation', function () {
|
||||
options: {
|
||||
sql: `
|
||||
SELECT
|
||||
cartodb_id,
|
||||
the_geom_webmercator,
|
||||
the_geom,
|
||||
value,
|
||||
@ -732,7 +955,13 @@ describe('aggregation', function () {
|
||||
});
|
||||
});
|
||||
|
||||
it('aggregates with full-sample placement', function (done) {
|
||||
it('aggregates with full-sample placement by default', function (done) {
|
||||
|
||||
// FIXME: skip until pg-mvt renderer is able to return all columns
|
||||
if (process.env.POSTGIS_VERSION === '2.4') {
|
||||
return done();
|
||||
}
|
||||
|
||||
this.mapConfig = createVectorMapConfig([
|
||||
{
|
||||
type: 'cartodb',
|
||||
@ -740,17 +969,6 @@ describe('aggregation', function () {
|
||||
sql: POINTS_SQL_1,
|
||||
resolution: 256,
|
||||
aggregation: {
|
||||
placement: 'full-sample',
|
||||
columns: {
|
||||
total: {
|
||||
aggregate_function: 'sum',
|
||||
aggregated_column: 'value'
|
||||
},
|
||||
v_avg: {
|
||||
aggregate_function: 'avg',
|
||||
aggregated_column: 'value'
|
||||
}
|
||||
},
|
||||
threshold: 1
|
||||
}
|
||||
}
|
||||
@ -847,10 +1065,10 @@ describe('aggregation', function () {
|
||||
}
|
||||
|
||||
assert.deepEqual(body, {
|
||||
errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample'],
|
||||
errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample'],
|
||||
errors_with_context:[{
|
||||
type: 'layer',
|
||||
message: 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample',
|
||||
message: 'Invalid placement. Valid values: centroid, point-grid, point-sample',
|
||||
layer: {
|
||||
id: "layer0",
|
||||
index: 0,
|
||||
|
||||
16
yarn.lock
16
yarn.lock
@ -244,6 +244,14 @@ carto@0.16.3:
|
||||
semver "^5.1.0"
|
||||
yargs "^4.2.0"
|
||||
|
||||
carto@CartoDB/carto#0.15.1-cdb1:
|
||||
version "0.15.1-cdb1"
|
||||
resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
|
||||
dependencies:
|
||||
mapnik-reference "~6.0.2"
|
||||
optimist "~0.6.0"
|
||||
underscore "~1.6.0"
|
||||
|
||||
carto@cartodb/carto#0.15.1-cdb3:
|
||||
version "0.15.1-cdb3"
|
||||
resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
|
||||
@ -252,14 +260,6 @@ carto@cartodb/carto#0.15.1-cdb3:
|
||||
optimist "~0.6.0"
|
||||
underscore "1.8.3"
|
||||
|
||||
"carto@github:cartodb/carto#0.15.1-cdb1":
|
||||
version "0.15.1-cdb1"
|
||||
resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
|
||||
dependencies:
|
||||
mapnik-reference "~6.0.2"
|
||||
optimist "~0.6.0"
|
||||
underscore "~1.6.0"
|
||||
|
||||
cartocolor@4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user