cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement aggregation filters

Browse Source
This commit is contained in:
Javier Goizueta 2018-03-21 17:01:32 +01:00
parent fefff3b788
commit 6ada8ba6a2
4 changed files with 680 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/cartodb/models/aggregation/aggregation-mapconfig.js
View File

@ -4,7 +4,8 @@ const aggregationValidator = require('./aggregation-validator');
const {
createPositiveNumberValidator,
createIncludesValueValidator,
createAggregationColumnsValidator
createAggregationColumnsValidator,
createAggregationFiltersValidator
} = aggregationValidator;
const SubstitutionTokens = require('../../utils/substitution-tokens');
@ -43,6 +44,18 @@ module.exports = class AggregationMapConfig extends MapConfig {
];
}
static get FILTER_PARAMETERS () {
return [
// TODO: valid combinations of parameters:
// * Except for less/greater params, only one parameter allowed per filter.
// * Any less parameter can be combined with one of the greater paramters. (to define a range)
'less_than', 'less_than_or_equal_to',
'greater_than', 'greater_than_or_equal_to',
'equal', 'not_equal',
'between', 'in', 'not_in'
];
}
static supportsGeometryType(geometryType) {
return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType);
}
@ -58,11 +71,15 @@ module.exports = class AggregationMapConfig extends MapConfig {
const positiveNumberValidator = createPositiveNumberValidator(this);
const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS);
const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS);
const aggregationFiltersValidator = createAggregationFiltersValidator(
this, AggregationMapConfig.FILTER_PARAMETERS
);
validate('resolution', positiveNumberValidator);
validate('placement', includesValidPlacementsValidator);
validate('threshold', positiveNumberValidator);
validate('columns', aggregationColumnsValidator);
validate('filters', aggregationFiltersValidator);
this.user = user;
this.pgConnection = connection;
@ -77,7 +94,8 @@ module.exports = class AggregationMapConfig extends MapConfig {
threshold = AggregationMapConfig.THRESHOLD,
placement,
columns = {},
dimensions = {}
dimensions = {},
filters = {}
} = this.getAggregation(index);
return aggregationQuery({
@ -87,6 +105,7 @@ module.exports = class AggregationMapConfig extends MapConfig {
placement,
columns,
dimensions,
filters,
isDefaultAggregation: this._isDefaultLayerAggregation(index)
});
}
@ -220,7 +239,8 @@ module.exports = class AggregationMapConfig extends MapConfig {
_isDefaultAggregation (aggregation) {
return aggregation.placement === undefined &&
aggregation.columns === undefined &&
this._isEmptyParameter(aggregation.dimensions);
this._isEmptyParameter(aggregation.dimensions) &&
this._isEmptyParameter(aggregation.filters);
}
_isEmptyParameter(parameter) {

129
lib/cartodb/models/aggregation/aggregation-query.js
View File

@ -42,7 +42,8 @@ const queryForOptions = (options) => templateForOptions(options)({
sourceQuery: options.query,
res: 256/options.resolution,
columns: options.columns,
dimensions: options.dimensions
dimensions: options.dimensions,
filters: options.filters
});
module.exports = queryForOptions;
@ -93,20 +94,23 @@ const aggregateColumnNames = (ctx, table) => {
return sep(Object.keys(columns));
};
const aggregateExpression = (column_name, column_parameters) => {
const aggregate_function = column_parameters.aggregate_function || 'count';
const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
if (!aggregate_definition) {
throw new Error("Invalid Aggregate function: '" + aggregate_function + "'");
}
return aggregate_definition.sql(column_name, column_parameters);
};
const aggregateColumnDefs = ctx => {
let columns = aggregateColumns(ctx);
return sep(Object.keys(columns).map(column_name => {
const aggregate_function = columns[column_name].aggregate_function || 'count';
const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
if (!aggregate_definition) {
throw new Error("Invalid Aggregate function: '" + aggregate_function + "'");
}
const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]);
const aggregate_expression = aggregateExpression(column_name, columns[column_name]);
return `${aggregate_expression} AS ${column_name}`;
}));
};
const aggregateDimensions = ctx => ctx.dimensions || {};
const dimensionNames = (ctx, table) => {
@ -127,6 +131,111 @@ const dimensionDefs = ctx => {
}));
};
const aggregateFilters = ctx => ctx.filters || {};
const filterConditionSQL = (expr, filter) => {
// TODO: validate filter parameters (e.g. cannot have both greater_than and greater_than or equal to)
if (filter) {
if (!Array.isArray(filter)) {
filter = [filter];
}
if (filter.length > 0) {
return filter.map(f => filterSingleConditionSQL(expr, f)).join(' OR ');
}
}
};
const filterSingleConditionSQL = (expr, filter) => {
let cond;
Object.keys(FILTERS).some(f => {
cond = FILTERS[f](expr, filter);
return cond;
});
return cond;
};
const sqlQ = (value) => {
if (isFinite(value)) {
return String(value);
}
return `'${value}'`; // TODO: escape single quotes! (by doubling them)
};
/* jshint eqeqeq: false */
/* x != null is used to check for both null and undefined; triple !== wouldn't do the trick */
const FILTERS = {
between: (expr, filter) => {
const lo = filter.greater_than_or_equal_to, hi = filter.less_than_or_equal_to;
if (lo != null && hi != null) {
return `(${expr} BETWEEN ${sqlQ(lo)} AND ${sqlQ(hi)})`;
}
},
in: (expr, filter) => {
if (filter.in != null) {
return `(${expr} IN (${filter.in.map(v => sqlQ(v)).join(',')}))`;
}
},
notin: (expr, filter) => {
if (filter.not_in != null) {
return `(${expr} NOT IN (${filter.not_in.map(v => sqlQ(v)).join(',')}))`;
}
},
equal: (expr, filter) => {
if (filter.equal != null) {
return `(${expr} = ${sqlQ(filter.equal)})`;
}
},
not_equal: (expr, filter) => {
if (filter.not_equal != null) {
return `(${expr} <> ${sqlQ(filter.not_equal)})`;
}
},
range: (expr, filter) => {
let conds = [];
if (filter.greater_than_or_equal_to != null) {
conds.push(`(${expr} >= ${sqlQ(filter.greater_than_or_equal_to)})`);
}
if (filter.greater_than != null) {
conds.push(`(${expr} > ${sqlQ(filter.greater_than)})`);
}
if (filter.less_than_or_equal_to != null) {
conds.push(`(${expr} <= ${sqlQ(filter.less_than_or_equal_to)})`);
}
if (filter.less_than != null) {
conds.push(`(${expr} < ${sqlQ(filter.less_than)})`);
}
if (conds.length > 0) {
return conds.join(' AND ');
}
}
};
const filterConditions = ctx => {
let columns = aggregateColumns(ctx);
let dimensions = aggregateDimensions(ctx);
let filters = aggregateFilters(ctx);
return Object.keys(filters).map(filtered_column => {
let filtered_expr;
if (columns[filtered_column]) {
filtered_expr = aggregateExpression(filtered_column, columns[filtered_column]);
}
else if (dimensions[filtered_column]) {
filtered_expr = dimensions[filtered_column];
}
if (!filtered_expr) {
throw new Error("Invalid filtered column: '" + filtered_column + "'");
}
return filterConditionSQL(filtered_expr, filters[filtered_column]);
}).join(' AND ');
};
const havingClause = ctx => {
let cond = filterConditions(ctx);
return cond ? `HAVING ${cond}` : '';
};
// SQL expression to compute the aggregation resolution (grid cell size).
// This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))`
// This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension
@ -164,6 +273,7 @@ const defaultAggregationQueryTemplate = ctx => `
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
${havingClause(ctx)}
) SELECT
_cdb_query.*
${aggregateColumnNames(ctx)}
@ -196,6 +306,7 @@ const aggregationQueryTemplates = {
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
${havingClause(ctx)}
`,
'point-grid': ctx => `
@ -214,6 +325,7 @@ const aggregationQueryTemplates = {
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE the_geom_webmercator && _cdb_params.bbox
GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
${havingClause(ctx)}
)
SELECT
row_number() over() AS cartodb_id,
@ -241,6 +353,7 @@ const aggregationQueryTemplates = {
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
${havingClause(ctx)}
)
SELECT
_cdb_clusters.cartodb_id,

32
lib/cartodb/models/aggregation/aggregation-validator.js
View File

@ -42,6 +42,38 @@ module.exports.createAggregationColumnsValidator = function (mapconfig, validAgg
};
};
module.exports.createAggregationFiltersValidator = function (mapconfig, validParameters) {
return function validateAggregationFilters (value, key, index) {
const dims = mapconfig.getAggregation(index).dimensions || {};
const cols = mapconfig.getAggregation(index).columns || {};
const validKeys = Object.keys(dims).concat(Object.keys(cols));
Object.keys(value).forEach((filteredName) => {
// filteredName must be the name of either an aggregated column or a dimension in the same layer
if (!validKeys.includes(filteredName)) {
const message = `Invalid filtered column: ${filteredName}`;
throw createLayerError(message, mapconfig, index);
}
// The filter parameters must be valid
let filters = value[filteredName];
// a single filter or an array of filters (to be OR-combined) are accepted
if (!Array.isArray(filters)) {
filters = [filters];
}
filters.forEach(params => {
Object.keys(params).forEach(paramName => {
if (!validParameters.includes(paramName)) {
const message = `Invalid filter parameter name: ${paramName}`;
throw createLayerError(message, mapconfig, index);
}
});
// TODO: check parameter value (params[paramName]) to be of the correct type
});
// TODO: if multiple parameters within params check the combination is valid,
// i.e. one of the *less* parameters and one of the *greater* parameters.
});
};
};
function createAggregationColumnNamesValidator(mapconfig) {
return function validateAggregationColumnNames (value, key, index) {
Object.keys(value).forEach((columnName) => {

504
test/acceptance/aggregation.js
View File

@ -1475,6 +1475,510 @@ describe('aggregation', function () {
done();
});
});
['centroid', 'point-sample', 'point-grid'].forEach(placement => {
it(`filters should work for ${placement} placement`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
placement: placement ,
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: {
greater_than_or_equal_to: 0
}
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value >= 0);
});
done();
});
});
});
['centroid', 'point-sample', 'point-grid'].forEach(placement => {
it(`multiple ORed filters should work for ${placement} placement`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
placement: placement ,
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: [
{ greater_than: 0 },
{ less_than: -2 }
]
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value > 0 || row.properties.value < -2);
});
done();
});
});
});
['centroid', 'point-sample', 'point-grid'].forEach(placement => {
it(`multiple ANDed filters should work for ${placement} placement`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_2,
aggregation: {
placement: placement ,
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
},
value2: {
aggregate_function: 'sum',
aggregated_column: 'sqrt_value'
}
},
filters: {
value: { greater_than: 0 },
value2: { less_than: 9 }
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value > 0 && row.properties.value2 < 9);
});
done();
});
});
});
it(`supports IN filters`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: { in: [1, 3] }
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value === 1 || row.properties.value === 3);
});
done();
});
});
it(`supports NOT IN filters`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: { not_in: [1, 3] }
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value !== 1 && row.properties.value !== 3);
});
done();
});
});
it(`supports EQUAL filters`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: [{ equal: 1}, { equal: 3}]
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value === 1 || row.properties.value === 3);
});
done();
});
});
it(`supports NOT EQUAL filters`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: { not_equal: 1 }
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value !== 1);
});
done();
});
});
it(`supports BETWEEN filters`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: {
greater_than_or_equal_to: -1,
less_than_or_equal_to: 2
}
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value >= -1 || row.properties.value <= 2);
});
done();
});
});
it(`supports RANGE filters`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: {
greater_than: -1,
less_than_or_equal_to: 2
}
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
format: 'mvt'
};
this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
if (err) {
return done(err);
}
const tileJSON = tile.toJSON();
tileJSON[0].features.forEach(row => {
assert.ok(row.properties.value > -1 || row.properties.value <= 2);
});
done();
});
});
it(`invalid filters cause errors`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: {
not_a_valid_parameter: 0
}
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Invalid filter parameter name: not_a_valid_parameter'],
errors_with_context:[{
type: 'layer',
message: 'Invalid filter parameter name: not_a_valid_parameter',
layer: {
id: "layer0",
index: 0,
type: "mapnik",
}
}]
});
done();
});
});
it(`filters on invalid columns cause errors`, function(done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
columns: {
value_sum: {
aggregate_function: 'sum',
aggregated_column: 'value'
}
},
filters: {
value: {
not_a_valid_parameter: 0
}
}
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Invalid filtered column: value'],
errors_with_context:[{
type: 'layer',
message: 'Invalid filtered column: value',
layer: {
id: "layer0",
index: 0,
type: "mapnik",
}
}]
});
done();
});
});
});
});
});