diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index 50e4dcfe..9306518c 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -4,7 +4,8 @@ const aggregationValidator = require('./aggregation-validator'); const { createPositiveNumberValidator, createIncludesValueValidator, - createAggregationColumnsValidator + createAggregationColumnsValidator, + createAggregationFiltersValidator } = aggregationValidator; const SubstitutionTokens = require('../../utils/substitution-tokens'); @@ -43,6 +44,18 @@ module.exports = class AggregationMapConfig extends MapConfig { ]; } + static get FILTER_PARAMETERS () { + return [ + // TODO: valid combinations of parameters: + // * Except for less/greater params, only one parameter allowed per filter. + // * Any less parameter can be combined with one of the greater paramters. (to define a range) + 'less_than', 'less_than_or_equal_to', + 'greater_than', 'greater_than_or_equal_to', + 'equal', 'not_equal', + 'between', 'in', 'not_in' + ]; + } + static supportsGeometryType(geometryType) { return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType); } @@ -58,11 +71,15 @@ module.exports = class AggregationMapConfig extends MapConfig { const positiveNumberValidator = createPositiveNumberValidator(this); const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS); const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS); + const aggregationFiltersValidator = createAggregationFiltersValidator( + this, AggregationMapConfig.FILTER_PARAMETERS + ); validate('resolution', positiveNumberValidator); validate('placement', includesValidPlacementsValidator); validate('threshold', positiveNumberValidator); validate('columns', aggregationColumnsValidator); + validate('filters', aggregationFiltersValidator); this.user = user; this.pgConnection = connection; @@ -77,7 +94,8 @@ module.exports = class AggregationMapConfig extends MapConfig { threshold = AggregationMapConfig.THRESHOLD, placement, columns = {}, - dimensions = {} + dimensions = {}, + filters = {} } = this.getAggregation(index); return aggregationQuery({ @@ -87,6 +105,7 @@ module.exports = class AggregationMapConfig extends MapConfig { placement, columns, dimensions, + filters, isDefaultAggregation: this._isDefaultLayerAggregation(index) }); } @@ -220,7 +239,8 @@ module.exports = class AggregationMapConfig extends MapConfig { _isDefaultAggregation (aggregation) { return aggregation.placement === undefined && aggregation.columns === undefined && - this._isEmptyParameter(aggregation.dimensions); + this._isEmptyParameter(aggregation.dimensions) && + this._isEmptyParameter(aggregation.filters); } _isEmptyParameter(parameter) { diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 16e897ab..bfd2def6 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -42,7 +42,8 @@ const queryForOptions = (options) => templateForOptions(options)({ sourceQuery: options.query, res: 256/options.resolution, columns: options.columns, - dimensions: options.dimensions + dimensions: options.dimensions, + filters: options.filters }); module.exports = queryForOptions; @@ -93,20 +94,23 @@ const aggregateColumnNames = (ctx, table) => { return sep(Object.keys(columns)); }; +const aggregateExpression = (column_name, column_parameters) => { + const aggregate_function = column_parameters.aggregate_function || 'count'; + const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; + if (!aggregate_definition) { + throw new Error("Invalid Aggregate function: '" + aggregate_function + "'"); + } + return aggregate_definition.sql(column_name, column_parameters); +}; + const aggregateColumnDefs = ctx => { let columns = aggregateColumns(ctx); return sep(Object.keys(columns).map(column_name => { - const aggregate_function = columns[column_name].aggregate_function || 'count'; - const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; - if (!aggregate_definition) { - throw new Error("Invalid Aggregate function: '" + aggregate_function + "'"); - } - const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]); + const aggregate_expression = aggregateExpression(column_name, columns[column_name]); return `${aggregate_expression} AS ${column_name}`; })); }; - const aggregateDimensions = ctx => ctx.dimensions || {}; const dimensionNames = (ctx, table) => { @@ -127,6 +131,111 @@ const dimensionDefs = ctx => { })); }; +const aggregateFilters = ctx => ctx.filters || {}; + +const filterConditionSQL = (expr, filter) => { + // TODO: validate filter parameters (e.g. cannot have both greater_than and greater_than or equal to) + + if (filter) { + if (!Array.isArray(filter)) { + filter = [filter]; + } + if (filter.length > 0) { + return filter.map(f => filterSingleConditionSQL(expr, f)).join(' OR '); + } + } +}; + +const filterSingleConditionSQL = (expr, filter) => { + let cond; + Object.keys(FILTERS).some(f => { + cond = FILTERS[f](expr, filter); + return cond; + }); + return cond; +}; + +const sqlQ = (value) => { + if (isFinite(value)) { + return String(value); + } + return `'${value}'`; // TODO: escape single quotes! (by doubling them) +}; + +/* jshint eqeqeq: false */ +/* x != null is used to check for both null and undefined; triple !== wouldn't do the trick */ + +const FILTERS = { + between: (expr, filter) => { + const lo = filter.greater_than_or_equal_to, hi = filter.less_than_or_equal_to; + if (lo != null && hi != null) { + return `(${expr} BETWEEN ${sqlQ(lo)} AND ${sqlQ(hi)})`; + } + }, + in: (expr, filter) => { + if (filter.in != null) { + return `(${expr} IN (${filter.in.map(v => sqlQ(v)).join(',')}))`; + } + }, + notin: (expr, filter) => { + if (filter.not_in != null) { + return `(${expr} NOT IN (${filter.not_in.map(v => sqlQ(v)).join(',')}))`; + } + }, + equal: (expr, filter) => { + if (filter.equal != null) { + return `(${expr} = ${sqlQ(filter.equal)})`; + } + }, + not_equal: (expr, filter) => { + if (filter.not_equal != null) { + return `(${expr} <> ${sqlQ(filter.not_equal)})`; + } + }, + range: (expr, filter) => { + let conds = []; + if (filter.greater_than_or_equal_to != null) { + conds.push(`(${expr} >= ${sqlQ(filter.greater_than_or_equal_to)})`); + } + if (filter.greater_than != null) { + conds.push(`(${expr} > ${sqlQ(filter.greater_than)})`); + } + if (filter.less_than_or_equal_to != null) { + conds.push(`(${expr} <= ${sqlQ(filter.less_than_or_equal_to)})`); + } + if (filter.less_than != null) { + conds.push(`(${expr} < ${sqlQ(filter.less_than)})`); + } + if (conds.length > 0) { + return conds.join(' AND '); + } + } +}; + +const filterConditions = ctx => { + let columns = aggregateColumns(ctx); + let dimensions = aggregateDimensions(ctx); + let filters = aggregateFilters(ctx); + return Object.keys(filters).map(filtered_column => { + let filtered_expr; + if (columns[filtered_column]) { + filtered_expr = aggregateExpression(filtered_column, columns[filtered_column]); + } + else if (dimensions[filtered_column]) { + filtered_expr = dimensions[filtered_column]; + } + if (!filtered_expr) { + throw new Error("Invalid filtered column: '" + filtered_column + "'"); + } + return filterConditionSQL(filtered_expr, filters[filtered_column]); + }).join(' AND '); +}; + +const havingClause = ctx => { + let cond = filterConditions(ctx); + return cond ? `HAVING ${cond}` : ''; +}; + // SQL expression to compute the aggregation resolution (grid cell size). // This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))` // This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension @@ -164,6 +273,7 @@ const defaultAggregationQueryTemplate = ctx => ` Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} + ${havingClause(ctx)} ) SELECT _cdb_query.* ${aggregateColumnNames(ctx)} @@ -196,6 +306,7 @@ const aggregationQueryTemplates = { Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} + ${havingClause(ctx)} `, 'point-grid': ctx => ` @@ -214,6 +325,7 @@ const aggregationQueryTemplates = { FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params WHERE the_geom_webmercator && _cdb_params.bbox GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)} + ${havingClause(ctx)} ) SELECT row_number() over() AS cartodb_id, @@ -241,6 +353,7 @@ const aggregationQueryTemplates = { Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} + ${havingClause(ctx)} ) SELECT _cdb_clusters.cartodb_id, diff --git a/lib/cartodb/models/aggregation/aggregation-validator.js b/lib/cartodb/models/aggregation/aggregation-validator.js index d0dc24c2..ce038f05 100644 --- a/lib/cartodb/models/aggregation/aggregation-validator.js +++ b/lib/cartodb/models/aggregation/aggregation-validator.js @@ -42,6 +42,38 @@ module.exports.createAggregationColumnsValidator = function (mapconfig, validAgg }; }; +module.exports.createAggregationFiltersValidator = function (mapconfig, validParameters) { + return function validateAggregationFilters (value, key, index) { + const dims = mapconfig.getAggregation(index).dimensions || {}; + const cols = mapconfig.getAggregation(index).columns || {}; + const validKeys = Object.keys(dims).concat(Object.keys(cols)); + Object.keys(value).forEach((filteredName) => { + // filteredName must be the name of either an aggregated column or a dimension in the same layer + if (!validKeys.includes(filteredName)) { + const message = `Invalid filtered column: ${filteredName}`; + throw createLayerError(message, mapconfig, index); + } + // The filter parameters must be valid + let filters = value[filteredName]; + // a single filter or an array of filters (to be OR-combined) are accepted + if (!Array.isArray(filters)) { + filters = [filters]; + } + filters.forEach(params => { + Object.keys(params).forEach(paramName => { + if (!validParameters.includes(paramName)) { + const message = `Invalid filter parameter name: ${paramName}`; + throw createLayerError(message, mapconfig, index); + } + }); + // TODO: check parameter value (params[paramName]) to be of the correct type + }); + // TODO: if multiple parameters within params check the combination is valid, + // i.e. one of the *less* parameters and one of the *greater* parameters. + }); + }; +}; + function createAggregationColumnNamesValidator(mapconfig) { return function validateAggregationColumnNames (value, key, index) { Object.keys(value).forEach((columnName) => { diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index d8870c8a..db03ad8e 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -1475,6 +1475,510 @@ describe('aggregation', function () { done(); }); }); + + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than_or_equal_to: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value >= 0); + }); + + done(); + }); + }); + }); + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`multiple ORed filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: [ + { greater_than: 0 }, + { less_than: -2 } + ] + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > 0 || row.properties.value < -2); + }); + + done(); + }); + }); + }); + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`multiple ANDed filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + }, + value2: { + aggregate_function: 'sum', + aggregated_column: 'sqrt_value' + } + }, + filters: { + value: { greater_than: 0 }, + value2: { less_than: 9 } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > 0 && row.properties.value2 < 9); + }); + + done(); + }); + }); + }); + + it(`supports IN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { in: [1, 3] } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value === 1 || row.properties.value === 3); + }); + + done(); + }); + }); + + it(`supports NOT IN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { not_in: [1, 3] } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value !== 1 && row.properties.value !== 3); + }); + + done(); + }); + }); + + it(`supports EQUAL filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: [{ equal: 1}, { equal: 3}] + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value === 1 || row.properties.value === 3); + }); + + done(); + }); + }); + + it(`supports NOT EQUAL filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { not_equal: 1 } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value !== 1); + }); + + done(); + }); + }); + + it(`supports BETWEEN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than_or_equal_to: -1, + less_than_or_equal_to: 2 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value >= -1 || row.properties.value <= 2); + }); + + done(); + }); + }); + + it(`supports RANGE filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than: -1, + less_than_or_equal_to: 2 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > -1 || row.properties.value <= 2); + }); + + done(); + }); + }); + + it(`invalid filters cause errors`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + not_a_valid_parameter: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + const options = { + response: { + status: 400 + } + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ 'Invalid filter parameter name: not_a_valid_parameter'], + errors_with_context:[{ + type: 'layer', + message: 'Invalid filter parameter name: not_a_valid_parameter', + layer: { + id: "layer0", + index: 0, + type: "mapnik", + } + }] + }); + + done(); + }); + }); + + it(`filters on invalid columns cause errors`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value_sum: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + not_a_valid_parameter: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + const options = { + response: { + status: 400 + } + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ 'Invalid filtered column: value'], + errors_with_context:[{ + type: 'layer', + message: 'Invalid filtered column: value', + layer: { + id: "layer0", + index: 0, + type: "mapnik", + } + }] + }); + + done(); + }); + }); + }); }); });