cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1150 from CartoDB/new-filters

Browse Source 
New dataview filters: circle & polygon
This commit is contained in:
Daniel G. Aubert 2019-12-20 09:40:09 +01:00 committed by GitHub
commit d63337f06f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 508 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -18,6 +18,7 @@ Announcements:
- Remove deprecated coverage tool istanbul, using nyc instead.
- Removed unused dockerfiles
- Use cartodb schema when using cartodb extension functions and tables.
- Implemented circle and polygon dataview filters.
## 8.0.0
Released 2019-11-13

2
lib/api/map/dataview-layergroup-controller.js
View File

@ -18,6 +18,8 @@ const ALLOWED_DATAVIEW_QUERY_PARAMS = [
'own_filter', // 0, 1
'no_filters', // 0, 1
'bbox', // w,s,e,n
'circle', // json
'polygon', // json
'start', // number
'end', // number
'column_type', // string

14
lib/backends/dataview.js
View File

@ -3,6 +3,8 @@
var _ = require('underscore');
var PSQL = require('cartodb-psql');
var BBoxFilter = require('../models/filter/bbox');
const CircleFilter = require('../models/filter/circle');
const PolygonFilter = require('../models/filter/polygon');
var DataviewFactory = require('../models/dataview/factory');
var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
const dbParamsFromReqParams = require('../utils/database-params');
@ -86,6 +88,12 @@ function getQueryWithFilters (dataviewDefinition, params) {
if (params.bbox) {
var bboxFilter = new BBoxFilter({ column: 'the_geom_webmercator', srid: 3857 }, { bbox: params.bbox });
query = bboxFilter.sql(query);
} else if (params.circle) {
const circleFilter = new CircleFilter({ column: 'the_geom_webmercator', srid: 3857 }, { circle: params.circle });
query = circleFilter.sql(query);
} else if (params.polygon) {
const polygonFilter = new PolygonFilter({ column: 'the_geom_webmercator', srid: 3857 }, { polygon: params.polygon });
query = polygonFilter.sql(query);
}
return query;
@ -197,6 +205,12 @@ function getQueryWithOwnFilters (dataviewDefinition, params) {
if (params.bbox) {
var bboxFilter = new BBoxFilter({ column: 'the_geom', srid: 4326 }, { bbox: params.bbox });
query = bboxFilter.sql(query);
} else if (params.circle) {
const circleFilter = new CircleFilter({ column: 'the_geom', srid: 4326 }, { circle: params.circle });
query = circleFilter.sql(query);
} else if (params.polygon) {
const polygonFilter = new PolygonFilter({ column: 'the_geom', srid: 4326 }, { polygon: params.polygon });
query = polygonFilter.sql(query);
}
return query;

66
lib/models/filter/circle.js Normal file
View File

@ -0,0 +1,66 @@
'use strict';
const debug = require('debug')('windshaft:filter:circle');
function filterQueryTpl ({ sql, column, srid, lng, lat, radiusInMeters } = {}) {
return `
SELECT
*
FROM (${sql}) _cdb_circle_filter
WHERE
ST_DWithin(
${srid === 4326 ? `${column}::geography` : `ST_Transform(${column}, 4326)::geography`},
ST_SetSRID(ST_Point(${lng}, ${lat}), 4326)::geography,
${radiusInMeters}
)
`;
}
module.exports = class CircleFilter {
constructor (filterDefinition, filterParams) {
const { circle } = filterParams;
let _circle;
if (!circle) {
const error = new Error('Circle filter expects to have a "circle" param');
error.type = 'filter';
throw error;
}
try {
_circle = JSON.parse(circle);
} catch (err) {
const error = new Error('Invalid circle parameter. Expected a valid JSON');
error.type = 'filter';
throw error;
}
const { lng, lat, radius } = _circle;
if (!Number.isFinite(lng) || !Number.isFinite(lat) || !Number.isFinite(radius)) {
const error = new Error('Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"');
error.type = 'filter';
throw error;
}
this.column = filterDefinition.column || 'the_geom_webmercator';
this.srid = filterDefinition.srid || 3857;
this.lng = lng;
this.lat = lat;
this.radius = radius;
}
sql (rawSql) {
const circleSql = filterQueryTpl({
sql: rawSql,
column: this.column,
srid: this.srid,
lng: this.lng,
lat: this.lat,
radiusInMeters: this.radius
});
debug(circleSql);
return circleSql;
}
};

71
lib/models/filter/polygon.js Normal file
View File

@ -0,0 +1,71 @@
'use strict';
const assert = require('assert');
const debug = require('debug')('windshaft:filter:polygon');
function filterQueryTpl ({ sql, column, srid, geojson } = {}) {
return `
SELECT
*
FROM (${sql}) _cdb_polygon_filter
WHERE
ST_Intersects(
${column},
ST_Transform(
ST_SetSRID(ST_GeomFromGeoJSON('${JSON.stringify(geojson)}'), 4326),
${srid}
)
)
`;
}
module.exports = class PolygonFilter {
constructor (filterDefinition, filterParams) {
const { polygon } = filterParams;
if (!polygon) {
const error = new Error('Polygon filter expects to have a "polygon" param');
error.type = 'filter';
throw error;
}
let geojson;
try {
geojson = JSON.parse(polygon);
} catch (err) {
const error = new Error('Invalid polygon parameter. Expected a valid GeoJSON');
error.type = 'filter';
throw error;
}
if (geojson.type !== 'Polygon') {
const error = new Error('Invalid type of geometry. Valid ones: "Polygon"');
error.type = 'filter';
throw error;
}
try {
const length = geojson.coordinates.length;
assert.deepStrictEqual(geojson.coordinates[0], geojson.coordinates[length - 1]);
} catch (error) {
throw new Error('Invalid geometry, it must be a closed polygon');
}
this.column = filterDefinition.column || 'the_geom_webmercator';
this.srid = filterDefinition.srid || 3857;
this.geojson = geojson;
}
sql (rawSql) {
const polygonSql = filterQueryTpl({
sql: rawSql,
column: this.column,
srid: this.srid,
geojson: this.geojson
});
debug(polygonSql);
return polygonSql;
}
};

353
test/acceptance/dataviews/spatial-filters-test.js Normal file
View File

@ -0,0 +1,353 @@
'use strict';
require('../../support/test-helper');
const assert = require('../../support/assert');
const TestClient = require('../../support/test-client');
describe('spatial filters', function () {
const mapConfig = {
version: '1.8.0',
layers: [
{
type: 'cartodb',
options: {
source: {
id: 'a0'
},
cartocss: '#points { marker-width: 10; marker-fill: red; }',
cartocss_version: '2.3.0'
}
}
],
dataviews: {
categories: {
source: {
id: 'a0'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'sum',
aggregationColumn: 'val'
}
},
counter: {
type: 'formula',
source: {
id: 'a0'
},
options: {
column: 'val',
operation: 'count'
}
}
},
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: `
SELECT
ST_SetSRID(ST_MakePoint(x, x), 4326) as the_geom,
ST_Transform(ST_SetSRID(ST_MakePoint(x, x), 4326), 3857) as the_geom_webmercator,
CASE
WHEN x % 4 = 0 THEN 1
WHEN x % 4 = 1 THEN 2
WHEN x % 4 = 2 THEN 3
ELSE 4
END AS val,
CASE
WHEN x % 4 = 0 THEN 'category_1'
WHEN x % 4 = 1 THEN 'category_2'
WHEN x % 4 = 2 THEN 'category_3'
ELSE 'category_4'
END AS cat
FROM generate_series(-10, 10) x
`
}
}
]
};
beforeEach(function (done) {
const apikey = 1234;
this.testClient = new TestClient(mapConfig, apikey);
done();
});
afterEach(function (done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
const scenarios = [
{
dataview: 'categories',
params: JSON.stringify({}),
expected: {
type: 'aggregation',
aggregation: 'sum',
count: 21,
nulls: 0,
nans: 0,
infinities: 0,
min: 5,
max: 40,
categoriesCount: 4,
categories: [
{ category: 'category_4', value: 40, agg: false },
{ category: 'category_3', value: 9, agg: false },
{ category: 'category_2', value: 6, agg: false },
{ category: 'category_1', value: 5, agg: false }
]
}
},
{
dataview: 'categories',
params: {
circle: JSON.stringify({
lat: 0,
lng: 0,
radius: 5000
})
},
expected: {
type: 'aggregation',
aggregation: 'sum',
count: 1,
nulls: 0,
nans: 0,
infinities: 0,
min: 1,
max: 1,
categoriesCount: 1,
categories: [
{ category: 'category_1', value: 1, agg: false }
]
}
}, {
dataview: 'categories',
params: {
circle: JSON.stringify({
lng: 0,
radius: 5000
}),
response: {
status: 400
}
},
expected: {
errors: [
'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"'
],
errors_with_context: [
{
type: 'filter',
message: 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"'
}
]
}
}, {
dataview: 'categories',
params: {
circle: JSON.stringify({
lat: 0,
radius: 5000
}),
response: {
status: 400
}
},
expected: {
errors: [
'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"'
],
errors_with_context: [
{
type: 'filter',
message: 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"'
}
]
}
}, {
dataview: 'categories',
params: {
circle: JSON.stringify({
lng: 0,
lat: 0
}),
response: {
status: 400
}
},
expected: {
errors: [
'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"'
],
errors_with_context: [
{
type: 'filter',
message: 'Missing parameter for Circle Filter, expected: "lng", "lat", and "radius"'
}
]
}
}, {
dataview: 'categories',
params: {
circle: 'wadus',
response: {
status: 400
}
},
expected: {
errors: [
'Invalid circle parameter. Expected a valid JSON'
],
errors_with_context: [
{
type: 'filter',
message: 'Invalid circle parameter. Expected a valid JSON'
}
]
}
},
{
dataview: 'categories',
params: {
polygon: JSON.stringify({
type: 'Polygon',
coordinates: [
[
[
-9.312286,
37.907367
],
[
11.969604,
6.487254
],
[
-32.217407,
6.957781
],
[
-9.312286,
37.907367
]
]
]
})
},
expected: {
type: 'aggregation',
aggregation: 'sum',
count: 3,
nulls: 0,
nans: 0,
infinities: 0,
min: 1,
max: 4,
categoriesCount: 3,
categories: [
{ category: 'category_4', value: 4, agg: false },
{ category: 'category_2', value: 2, agg: false },
{ category: 'category_1', value: 1, agg: false }
]
}
}, {
dataview: 'categories',
params: {
polygon: 'wadus',
response: {
status: 400
}
},
expected: {
errors: [
'Invalid polygon parameter. Expected a valid GeoJSON'
],
errors_with_context: [
{
type: 'filter',
message: 'Invalid polygon parameter. Expected a valid GeoJSON'
}
]
}
}, {
dataview: 'categories',
params: {
polygon: JSON.stringify({
type: 'Point',
coordinates: [30, 10]
}),
response: {
status: 400
}
},
expected: {
errors: [
'Invalid type of geometry. Valid ones: "Polygon"'
],
errors_with_context: [
{
type: 'filter',
message: 'Invalid type of geometry. Valid ones: "Polygon"'
}
]
}
}, {
dataview: 'categories',
params: {
polygon: JSON.stringify({
type: 'Polygon',
coordinates: [[[]]]
}),
response: {
status: 400
}
},
expected: {
errors: [
'Too few ordinates in GeoJSON'
],
errors_with_context: [
{
type: 'unknown',
message: 'Too few ordinates in GeoJSON'
}
]
}
},
{
dataview: 'counter',
params: {
circle: JSON.stringify({
lat: 0,
lng: 0,
radius: 50000
})
},
expected: {
nulls: 0,
operation: 'count',
result: 1,
type: 'formula'
}
}
];
scenarios.forEach(function (scenario) {
it(`should get aggregation dataview with params: ${JSON.stringify(scenario.params)}`, function (done) {
this.testClient.getDataview(scenario.dataview, scenario.params, (err, dataview) => {
assert.ifError(err);
assert.deepStrictEqual(dataview, scenario.expected);
done();
});
});
});
});

2
test/support/test-client.js
View File

@ -485,7 +485,7 @@ TestClient.prototype.getDataview = function (dataviewName, params, callback) {
urlParams.own_filter = params.own_filter;
}
['bbox', 'bins', 'start', 'end', 'aggregation', 'offset', 'categories'].forEach(function (extraParam) {
['bbox', 'circle', 'polygon', 'bins', 'start', 'end', 'aggregation', 'offset', 'categories'].forEach(function (extraParam) {
if (Object.prototype.hasOwnProperty.call(params, extraParam)) {
urlParams[extraParam] = params[extraParam];
}