diff --git a/NEWS.md b/NEWS.md index d5ce679c..d3aae5fb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,6 +18,7 @@ Announcements: - Remove deprecated coverage tool istanbul, using nyc instead. - Removed unused dockerfiles - Use cartodb schema when using cartodb extension functions and tables. +- Implemented circle and polygon dataview filters. ## 8.0.0 Released 2019-11-13 diff --git a/lib/api/map/dataview-layergroup-controller.js b/lib/api/map/dataview-layergroup-controller.js index a10fbc25..8f849565 100644 --- a/lib/api/map/dataview-layergroup-controller.js +++ b/lib/api/map/dataview-layergroup-controller.js @@ -18,6 +18,8 @@ const ALLOWED_DATAVIEW_QUERY_PARAMS = [ 'own_filter', // 0, 1 'no_filters', // 0, 1 'bbox', // w,s,e,n + 'circle', // json + 'polygon', // json 'start', // number 'end', // number 'column_type', // string diff --git a/lib/backends/dataview.js b/lib/backends/dataview.js index 080718e1..f3d05f73 100644 --- a/lib/backends/dataview.js +++ b/lib/backends/dataview.js @@ -3,6 +3,8 @@ var _ = require('underscore'); var PSQL = require('cartodb-psql'); var BBoxFilter = require('../models/filter/bbox'); +const CircleFilter = require('../models/filter/circle'); +const PolygonFilter = require('../models/filter/polygon'); var DataviewFactory = require('../models/dataview/factory'); var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory'); const dbParamsFromReqParams = require('../utils/database-params'); @@ -86,6 +88,12 @@ function getQueryWithFilters (dataviewDefinition, params) { if (params.bbox) { var bboxFilter = new BBoxFilter({ column: 'the_geom_webmercator', srid: 3857 }, { bbox: params.bbox }); query = bboxFilter.sql(query); + } else if (params.circle) { + const circleFilter = new CircleFilter({ column: 'the_geom_webmercator', srid: 3857 }, { circle: params.circle }); + query = circleFilter.sql(query); + } else if (params.polygon) { + const polygonFilter = new PolygonFilter({ column: 'the_geom_webmercator', srid: 3857 }, { polygon: params.polygon }); + query = polygonFilter.sql(query); } return query; @@ -197,6 +205,12 @@ function getQueryWithOwnFilters (dataviewDefinition, params) { if (params.bbox) { var bboxFilter = new BBoxFilter({ column: 'the_geom', srid: 4326 }, { bbox: params.bbox }); query = bboxFilter.sql(query); + } else if (params.circle) { + const circleFilter = new CircleFilter({ column: 'the_geom', srid: 4326 }, { circle: params.circle }); + query = circleFilter.sql(query); + } else if (params.polygon) { + const polygonFilter = new PolygonFilter({ column: 'the_geom', srid: 4326 }, { polygon: params.polygon }); + query = polygonFilter.sql(query); } return query; diff --git a/lib/models/filter/circle.js b/lib/models/filter/circle.js new file mode 100644 index 00000000..b09b6446 --- /dev/null +++ b/lib/models/filter/circle.js @@ -0,0 +1,66 @@ +'use strict'; + +const debug = require('debug')('windshaft:filter:circle'); +function filterQueryTpl ({ sql, column, srid, lng, lat, radiusInMeters } = {}) { + return ` + SELECT + * + FROM (${sql}) _cdb_circle_filter + WHERE + ST_DWithin( + ${srid === 4326 ? `${column}::geography` : `ST_Transform(${column}, 4326)::geography`}, + ST_SetSRID(ST_Point(${lng}, ${lat}), 4326)::geography, + ${radiusInMeters} + ) + `; +} + +module.exports = class CircleFilter { + constructor (filterDefinition, filterParams) { + const { circle } = filterParams; + let _circle; + + if (!circle) { + const error = new Error('Circle filter expects to have a "circle" param'); + error.type = 'filter'; + throw error; + } + + try { + _circle = JSON.parse(circle); + } catch (err) { + const error = new Error('Invalid circle parameter. Expected a valid JSON'); + error.type = 'filter'; + throw error; + } + + const { lng, lat, radius } = _circle; + + if (!Number.isFinite(lng) || !Number.isFinite(lat) || !Number.isFinite(radius)) { + const error = new Error('Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"'); + error.type = 'filter'; + throw error; + } + + this.column = filterDefinition.column || 'the_geom_webmercator'; + this.srid = filterDefinition.srid || 3857; + this.lng = lng; + this.lat = lat; + this.radius = radius; + } + + sql (rawSql) { + const circleSql = filterQueryTpl({ + sql: rawSql, + column: this.column, + srid: this.srid, + lng: this.lng, + lat: this.lat, + radiusInMeters: this.radius + }); + + debug(circleSql); + + return circleSql; + } +}; diff --git a/lib/models/filter/polygon.js b/lib/models/filter/polygon.js new file mode 100644 index 00000000..357f00bf --- /dev/null +++ b/lib/models/filter/polygon.js @@ -0,0 +1,71 @@ +'use strict'; + +const assert = require('assert'); +const debug = require('debug')('windshaft:filter:polygon'); +function filterQueryTpl ({ sql, column, srid, geojson } = {}) { + return ` + SELECT + * + FROM (${sql}) _cdb_polygon_filter + WHERE + ST_Intersects( + ${column}, + ST_Transform( + ST_SetSRID(ST_GeomFromGeoJSON('${JSON.stringify(geojson)}'), 4326), + ${srid} + ) + ) + `; +} + +module.exports = class PolygonFilter { + constructor (filterDefinition, filterParams) { + const { polygon } = filterParams; + + if (!polygon) { + const error = new Error('Polygon filter expects to have a "polygon" param'); + error.type = 'filter'; + throw error; + } + + let geojson; + + try { + geojson = JSON.parse(polygon); + } catch (err) { + const error = new Error('Invalid polygon parameter. Expected a valid GeoJSON'); + error.type = 'filter'; + throw error; + } + + if (geojson.type !== 'Polygon') { + const error = new Error('Invalid type of geometry. Valid ones: "Polygon"'); + error.type = 'filter'; + throw error; + } + + try { + const length = geojson.coordinates.length; + assert.deepStrictEqual(geojson.coordinates[0], geojson.coordinates[length - 1]); + } catch (error) { + throw new Error('Invalid geometry, it must be a closed polygon'); + } + + this.column = filterDefinition.column || 'the_geom_webmercator'; + this.srid = filterDefinition.srid || 3857; + this.geojson = geojson; + } + + sql (rawSql) { + const polygonSql = filterQueryTpl({ + sql: rawSql, + column: this.column, + srid: this.srid, + geojson: this.geojson + }); + + debug(polygonSql); + + return polygonSql; + } +}; diff --git a/test/acceptance/dataviews/spatial-filters-test.js b/test/acceptance/dataviews/spatial-filters-test.js new file mode 100644 index 00000000..829829da --- /dev/null +++ b/test/acceptance/dataviews/spatial-filters-test.js @@ -0,0 +1,353 @@ +'use strict'; + +require('../../support/test-helper'); + +const assert = require('../../support/assert'); +const TestClient = require('../../support/test-client'); + +describe('spatial filters', function () { + const mapConfig = { + version: '1.8.0', + layers: [ + { + type: 'cartodb', + options: { + source: { + id: 'a0' + }, + cartocss: '#points { marker-width: 10; marker-fill: red; }', + cartocss_version: '2.3.0' + } + } + ], + dataviews: { + categories: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } + }, + counter: { + type: 'formula', + source: { + id: 'a0' + }, + options: { + column: 'val', + operation: 'count' + } + } + }, + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: ` + SELECT + ST_SetSRID(ST_MakePoint(x, x), 4326) as the_geom, + ST_Transform(ST_SetSRID(ST_MakePoint(x, x), 4326), 3857) as the_geom_webmercator, + CASE + WHEN x % 4 = 0 THEN 1 + WHEN x % 4 = 1 THEN 2 + WHEN x % 4 = 2 THEN 3 + ELSE 4 + END AS val, + CASE + WHEN x % 4 = 0 THEN 'category_1' + WHEN x % 4 = 1 THEN 'category_2' + WHEN x % 4 = 2 THEN 'category_3' + ELSE 'category_4' + END AS cat + FROM generate_series(-10, 10) x + ` + } + } + ] + }; + + beforeEach(function (done) { + const apikey = 1234; + this.testClient = new TestClient(mapConfig, apikey); + done(); + }); + + afterEach(function (done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + const scenarios = [ + { + dataview: 'categories', + params: JSON.stringify({}), + expected: { + type: 'aggregation', + aggregation: 'sum', + count: 21, + nulls: 0, + nans: 0, + infinities: 0, + min: 5, + max: 40, + categoriesCount: 4, + categories: [ + { category: 'category_4', value: 40, agg: false }, + { category: 'category_3', value: 9, agg: false }, + { category: 'category_2', value: 6, agg: false }, + { category: 'category_1', value: 5, agg: false } + ] + } + }, + { + dataview: 'categories', + params: { + circle: JSON.stringify({ + lat: 0, + lng: 0, + radius: 5000 + }) + }, + expected: { + type: 'aggregation', + aggregation: 'sum', + count: 1, + nulls: 0, + nans: 0, + infinities: 0, + min: 1, + max: 1, + categoriesCount: 1, + categories: [ + { category: 'category_1', value: 1, agg: false } + ] + } + }, { + dataview: 'categories', + params: { + circle: JSON.stringify({ + lng: 0, + radius: 5000 + }), + response: { + status: 400 + } + }, + expected: { + errors: [ + 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"' + ], + errors_with_context: [ + { + type: 'filter', + message: 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"' + } + ] + } + }, { + dataview: 'categories', + params: { + circle: JSON.stringify({ + lat: 0, + radius: 5000 + }), + response: { + status: 400 + } + }, + expected: { + errors: [ + 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"' + ], + errors_with_context: [ + { + type: 'filter', + message: 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"' + } + ] + } + }, { + dataview: 'categories', + params: { + circle: JSON.stringify({ + lng: 0, + lat: 0 + }), + response: { + status: 400 + } + }, + expected: { + errors: [ + 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"' + ], + errors_with_context: [ + { + type: 'filter', + message: 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"' + } + ] + } + }, { + dataview: 'categories', + params: { + circle: 'wadus', + response: { + status: 400 + } + }, + expected: { + errors: [ + 'Invalid circle parameter. Expected a valid JSON' + ], + errors_with_context: [ + { + type: 'filter', + message: 'Invalid circle parameter. Expected a valid JSON' + } + ] + } + }, + { + dataview: 'categories', + params: { + polygon: JSON.stringify({ + type: 'Polygon', + coordinates: [ + [ + [ + -9.312286, + 37.907367 + ], + [ + 11.969604, + 6.487254 + ], + [ + -32.217407, + 6.957781 + ], + [ + -9.312286, + 37.907367 + ] + ] + ] + }) + }, + expected: { + type: 'aggregation', + aggregation: 'sum', + count: 3, + nulls: 0, + nans: 0, + infinities: 0, + min: 1, + max: 4, + categoriesCount: 3, + categories: [ + { category: 'category_4', value: 4, agg: false }, + { category: 'category_2', value: 2, agg: false }, + { category: 'category_1', value: 1, agg: false } + ] + } + }, { + dataview: 'categories', + params: { + polygon: 'wadus', + response: { + status: 400 + } + }, + expected: { + errors: [ + 'Invalid polygon parameter. Expected a valid GeoJSON' + ], + errors_with_context: [ + { + type: 'filter', + message: 'Invalid polygon parameter. Expected a valid GeoJSON' + } + ] + } + }, { + dataview: 'categories', + params: { + polygon: JSON.stringify({ + type: 'Point', + coordinates: [30, 10] + }), + response: { + status: 400 + } + }, + expected: { + errors: [ + 'Invalid type of geometry. Valid ones: "Polygon"' + ], + errors_with_context: [ + { + type: 'filter', + message: 'Invalid type of geometry. Valid ones: "Polygon"' + } + ] + } + }, { + dataview: 'categories', + params: { + polygon: JSON.stringify({ + type: 'Polygon', + coordinates: [[[]]] + }), + response: { + status: 400 + } + }, + expected: { + errors: [ + 'Too few ordinates in GeoJSON' + ], + errors_with_context: [ + { + type: 'unknown', + message: 'Too few ordinates in GeoJSON' + } + ] + } + }, + { + dataview: 'counter', + params: { + circle: JSON.stringify({ + lat: 0, + lng: 0, + radius: 50000 + }) + }, + expected: { + nulls: 0, + operation: 'count', + result: 1, + type: 'formula' + } + } + ]; + + scenarios.forEach(function (scenario) { + it(`should get aggregation dataview with params: ${JSON.stringify(scenario.params)}`, function (done) { + this.testClient.getDataview(scenario.dataview, scenario.params, (err, dataview) => { + assert.ifError(err); + assert.deepStrictEqual(dataview, scenario.expected); + done(); + }); + }); + }); +}); diff --git a/test/support/test-client.js b/test/support/test-client.js index 634509c0..6f465295 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -485,7 +485,7 @@ TestClient.prototype.getDataview = function (dataviewName, params, callback) { urlParams.own_filter = params.own_filter; } - ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset', 'categories'].forEach(function (extraParam) { + ['bbox', 'circle', 'polygon', 'bins', 'start', 'end', 'aggregation', 'offset', 'categories'].forEach(function (extraParam) { if (Object.prototype.hasOwnProperty.call(params, extraParam)) { urlParams[extraParam] = params[extraParam]; }