cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into full-sample

Browse Source
This commit is contained in:
Daniel García Aubert 2017-12-19 12:47:36 +01:00
commit 94a5e66881
11 changed files with 1083 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
NEWS.md
View File

@ -1,9 +1,18 @@
# Changelog
## 4.5.0
Released 2017-mm-dd
## 4.6.0
Released YYYY-MM-DD
Announcements:
- Validate aggregation input params
## 4.5.0
Released 2017-12-19
Announcements:
- Date histograms: Add second, decade, century and millenium aggregations
- Date histograms: Switch the auto threshold from 366 buckets to 100.
- Logging all errors.
- Add support for aggregated visualizations.
- Allow vector-only map-config creation.

34
lib/cartodb/models/aggregation/aggregation-map-config.js
View File

@ -1,34 +0,0 @@
const MapConfig = require('windshaft').model.MapConfig;
module.exports = class AggregationMapConfig extends MapConfig {
constructor (config, datasource) {
super(config, datasource);
}
isAggregationMapConfig () {
return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation();
}
isAggregationLayer (index) {
return this.isVectorOnlyMapConfig() || this.hasLayerAggregation(index);
}
hasAnyLayerAggregation () {
const layers = this.getLayers();
for (let index = 0; index < layers.length; index++) {
if (this.hasLayerAggregation(index)) {
return true;
}
}
return false;
}
hasLayerAggregation (index) {
const layer = this.getLayer(index);
const { aggregation } = layer.options;
return aggregation !== undefined && (typeof aggregation === 'object' || typeof aggregation === 'boolean');
}
};

124
lib/cartodb/models/aggregation/aggregation-mapconfig.js Normal file
View File

@ -0,0 +1,124 @@
const MapConfig = require('windshaft').model.MapConfig;
const aggregationQuery = require('./aggregation-query');
const { SUPPORTED_AGGREGATE_FUNCTIONS } = require('./aggregation-query');
const aggregationValidator = require('./aggregation-validator');
const {
createPositiveNumberValidator,
createIncludesValueValidator,
createAggregationColumnsValidator
} = aggregationValidator;
module.exports = class AggregationMapConfig extends MapConfig {
static get PLACEMENTS () {
return [
'centroid',
'point-grid',
'point-sample'
];
}
static get PLACEMENT () {
return AggregationMapConfig.PLACEMENTS[0];
}
static get THRESHOLD () {
return 1e5; // 100K
}
static get RESOLUTION () {
return 1;
}
static get SUPPORTED_GEOMETRY_TYPES () {
return [
'ST_Point'
];
}
static supportsGeometryType(geometryType) {
return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType);
}
constructor (config, datasource) {
super(config, datasource);
const validate = aggregationValidator(this);
const positiveNumberValidator = createPositiveNumberValidator(this);
const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS);
const aggregationColumnsValidator = createAggregationColumnsValidator(this, SUPPORTED_AGGREGATE_FUNCTIONS);
validate('resolution', positiveNumberValidator);
validate('placement', includesValidPlacementsValidator);
validate('threshold', positiveNumberValidator);
validate('columns', aggregationColumnsValidator);
}
getAggregatedQuery (index) {
const { sql_raw, sql } = this.getLayer(index).options;
const {
resolution = AggregationMapConfig.RESOLUTION,
threshold = AggregationMapConfig.THRESHOLD,
placement = AggregationMapConfig.PLACEMENT,
columns = {},
dimmensions = {}
} = this.getAggregation(index);
return aggregationQuery({
query: sql_raw || sql,
resolution,
threshold,
placement,
columns,
dimmensions
});
}
isAggregationMapConfig () {
return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation();
}
isAggregationLayer (index) {
return this.isVectorOnlyMapConfig() || this.hasLayerAggregation(index);
}
hasAnyLayerAggregation () {
const layers = this.getLayers();
for (let index = 0; index < layers.length; index++) {
if (this.hasLayerAggregation(index)) {
return true;
}
}
return false;
}
hasLayerAggregation (index) {
const layer = this.getLayer(index);
const { aggregation } = layer.options;
return aggregation !== undefined && (typeof aggregation === 'object' || typeof aggregation === 'boolean');
}
getAggregation (index) {
if (!this.hasLayerAggregation(index)) {
return;
}
const { aggregation } = this.getLayer(index).options;
if (typeof aggregation === 'boolean') {
return {};
}
return aggregation;
}
doesLayerReachThreshold(index, featureCount) {
const threshold = this.getAggregation(index) && this.getAggregation(index).threshold ?
this.getAggregation(index).threshold :
AggregationMapConfig.THRESHOLD;
return featureCount >= threshold;
}
};

2
lib/cartodb/models/aggregation/aggregation-query.js
View File

@ -56,6 +56,8 @@ const SUPPORTED_AGGREGATE_FUNCTIONS = {
}
};
module.exports.SUPPORTED_AGGREGATE_FUNCTIONS = Object.keys(SUPPORTED_AGGREGATE_FUNCTIONS);
const sep = (list) => {
let expr = list.join(', ');
return expr ? ', ' + expr : expr;

93
lib/cartodb/models/aggregation/aggregation-validator.js Normal file
View File

@ -0,0 +1,93 @@
module.exports = function aggregationValidator (mapconfig) {
return function validateProperty (key, validator) {
for (let index = 0; index < mapconfig.getLayers().length; index++) {
const aggregation = mapconfig.getAggregation(index);
if (aggregation === undefined || aggregation[key] === undefined) {
continue;
}
validator(aggregation[key], key, index);
}
};
};
module.exports.createIncludesValueValidator = function (mapconfig, validValues) {
return function validateIncludesValue (value, key, index) {
if (!validValues.includes(value)) {
const message = `Invalid ${key}. Valid values: ${validValues.join(', ')}`;
throw createLayerError(message, mapconfig, index);
}
};
};
module.exports.createPositiveNumberValidator = function (mapconfig) {
return function validatePositiveNumber (value, key, index) {
if (!Number.isFinite(value) || value <= 0) {
const message = `Invalid ${key}, should be a number greather than 0`;
throw createLayerError(message, mapconfig, index);
}
};
};
module.exports.createAggregationColumnsValidator = function (mapconfig, validAggregatedFunctions) {
const validateAggregationColumnNames = createAggregationColumnNamesValidator(mapconfig);
const validateAggregateFunction = createAggregateFunctionValidator(mapconfig, validAggregatedFunctions);
const validateAggregatedColumn = createAggregatedColumnValidator(mapconfig);
return function validateAggregationColumns (value, key, index) {
validateAggregationColumnNames(value, key, index);
validateAggregateFunction(value, key, index);
validateAggregatedColumn(value, key, index);
};
};
function createAggregationColumnNamesValidator(mapconfig) {
return function validateAggregationColumnNames (value, key, index) {
Object.keys(value).forEach((columnName) => {
if (columnName.length <= 0) {
const message = `Invalid column name, should be a non empty string`;
throw createLayerError(message, mapconfig, index);
}
});
};
}
function createAggregateFunctionValidator (mapconfig, validAggregatedFunctions) {
return function validateAggregateFunction (value, key, index) {
Object.keys(value).forEach((columnName) => {
const { aggregate_function } = value[columnName];
if (!validAggregatedFunctions.includes(aggregate_function)) {
const message = `Unsupported aggregation function ${aggregate_function},` +
` valid ones: ${validAggregatedFunctions.join(', ')}`;
throw createLayerError(message, mapconfig, index);
}
});
};
}
function createAggregatedColumnValidator (mapconfig) {
return function validateAggregatedColumn (value, key, index) {
Object.keys(value).forEach((columnName) => {
const { aggregated_column } = value[columnName];
if (typeof aggregated_column !== 'string' || aggregated_column <= 0) {
const message = `Invalid aggregated column, should be a non empty string`;
throw createLayerError(message, mapconfig, index);
}
});
};
}
function createLayerError(message, mapconfig, index) {
const error = new Error(message);
error.type = 'layer';
error.layer = {
id: mapconfig.getLayerId(index),
index: index,
type: mapconfig.layerType(index)
};
return error;
}

26
lib/cartodb/models/aggregation/aggregation.js
View File

@ -1,26 +0,0 @@
const aggregationQuery = require('./aggregation-query');
module.exports = class Aggregation {
static get THRESHOLD() {
return 1e5; // 100K
}
constructor (mapconfig, query, {
resolution = 1,
threshold = Aggregation.THRESHOLD,
placement = 'centroid',
columns = {},
dimensions = {}
} = {}) {
this.mapconfig = mapconfig;
this.query = query;
this.resolution = resolution;
this.threshold = threshold;
this.placement = placement;
this.columns = columns;
this.dimensions = dimensions;
}
sql () {
return aggregationQuery(this);
}
};

21
lib/cartodb/models/dataview/histograms/date-histogram.js
View File

@ -127,8 +127,11 @@ const dateIntervalQueryTpl = ctx => `
FROM __cdb_interval_in_minutes, __cdb_dates
)
SELECT
ROUND(__cdb_days / 365243) AS millennium,
ROUND(__cdb_days / 36525) AS century,
ROUND(__cdb_days / 3652) AS decade,
ROUND(__cdb_days / 365) AS year,
ROUND(__cdb_days / 90) AS quarter,
ROUND(__cdb_days / 91) AS quarter,
ROUND(__cdb_days / 30) AS month,
ROUND(__cdb_days / 7) AS week,
__cdb_days AS day,
@ -138,18 +141,22 @@ const dateIntervalQueryTpl = ctx => `
FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
`;
const MAX_INTERVAL_VALUE = 366;
/** Constant to switch between aggregations in auto mode */
const MAX_INTERVAL_VALUE = 100;
const DATE_AGGREGATIONS = {
'auto': true,
'second' : true,
'minute': true,
'hour': true,
'day': true,
'week': true,
'month': true,
'quarter': true,
'year': true
'year': true,
'decade' : true,
'century' : true,
'millennium' : true
};
/**
@ -243,9 +250,9 @@ ORDER BY bin ASC;
return callback(err);
}
const aggegations = result.rows[0];
const aggregation = Object.keys(aggegations)
.map(key => ({ name: key, value: aggegations[key] }))
const aggregations = result.rows[0];
const aggregation = Object.keys(aggregations)
.map(key => ({ name: key, value: aggregations[key] }))
.reduce((closer, current) => {
if (current.value > MAX_INTERVAL_VALUE) {
return closer;

54
lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js
View File

@ -1,9 +1,9 @@
const Aggregation = require('../../aggregation/aggregation');
const AggregationMapConfig = require('../../aggregation/aggregation-map-config');
const AggregationMapConfig = require('../../aggregation/aggregation-mapconfig');
const queryUtils = require('../../../utils/query-utils');
const unsupportedGeometryTypeErrorMessage = ctx =>
`Unsupported geometry type: ${ctx.geometryType}. Aggregation is available only for geometry type: ST_Point`;
`Unsupported geometry type: ${ctx.geometryType}. ` +
`Aggregation is available only for geometry type: ${AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES}`;
const invalidAggregationParamValueErrorMessage = ctx =>
`Invalid value for 'aggregation' query param: ${ctx.value}. Valid ones are 'true' or 'false'`;
@ -14,11 +14,17 @@ module.exports = class AggregationMapConfigAdapter {
}
getMapConfig (user, requestMapConfig, params, context, callback) {
if (!this._isValidAggregationParam(params)) {
if (!this._isValidAggregationQueryParam(params)) {
return callback(new Error(invalidAggregationParamValueErrorMessage({ value: params.aggregation })));
}
const mapConfig = new AggregationMapConfig(requestMapConfig);
let mapConfig;
try {
mapConfig = new AggregationMapConfig(requestMapConfig);
} catch (err) {
return callback(err);
}
if (!this._shouldAdapt(mapConfig, params)) {
return callback(null, requestMapConfig);
@ -33,7 +39,7 @@ module.exports = class AggregationMapConfigAdapter {
});
}
_isValidAggregationParam (params) {
_isValidAggregationQueryParam (params) {
const { aggregation } = params;
return aggregation === undefined || aggregation === 'true' || aggregation === 'false';
}
@ -84,11 +90,9 @@ module.exports = class AggregationMapConfigAdapter {
}
if (shouldAdapt) {
const sql = layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql;
const aggregation = new Aggregation(mapConfig, sql, layer.options.aggregation);
const sqlQueryWrap = layer.options.sql_wrap;
let aggregationSql = aggregation.sql();
let aggregationSql = mapConfig.getAggregatedQuery(index);
if (sqlQueryWrap) {
aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql);
@ -103,10 +107,8 @@ module.exports = class AggregationMapConfigAdapter {
}
_shouldAdaptLayer (connection, mapConfig, layer, index, callback) {
let shouldAdapt = false;
if (!mapConfig.isAggregationLayer(index)) {
return callback(null, shouldAdapt);
return callback(null, false);
}
const aggregationMetadata = queryUtils.getAggregationMetadata({
@ -115,29 +117,29 @@ module.exports = class AggregationMapConfigAdapter {
connection.query(aggregationMetadata, (err, res) => {
if (err) {
return callback(null, shouldAdapt);
return callback(null, false);
}
const result = res.rows[0] || {};
const estimatedFeatureCount = result.count;
const threshold = layer.options.aggregation && layer.options.aggregation.threshold ?
layer.options.aggregation.threshold :
Aggregation.THRESHOLD;
if (!AggregationMapConfig.supportsGeometryType(result.type)) {
const message = unsupportedGeometryTypeErrorMessage({ geometryType: result.type });
const error = new Error(message);
error.type = 'layer';
error.layer = {
id: mapConfig.getLayerId(index),
index: index,
type: mapConfig.layerType(index)
};
if (estimatedFeatureCount < threshold) {
return callback(null, shouldAdapt);
return callback(error);
}
const geometryType = result.type;
if (geometryType !== 'ST_Point') {
return callback(new Error(unsupportedGeometryTypeErrorMessage({ geometryType })));
if (!mapConfig.doesLayerReachThreshold(index, result.count)) {
return callback(null, false);
}
shouldAdapt = true;
callback(null, shouldAdapt);
callback(null, true);
});
}

2
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "4.4.1",
"version": "4.5.0",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"

284
test/acceptance/aggregation.js
View File

@ -456,9 +456,14 @@ describe('aggregation', function () {
' Aggregation is available only for geometry type: ST_Point'
],
errors_with_context:[{
type: 'unknown',
type: 'layer',
message: 'Unsupported geometry type: ST_Polygon.' +
' Aggregation is available only for geometry type: ST_Point'
' Aggregation is available only for geometry type: ST_Point',
layer: {
id: 'layer0',
index: 0,
type: 'mapnik'
}
}]
});
@ -686,6 +691,281 @@ describe('aggregation', function () {
done();
});
});
it('should fail with bad resolution', function (done) {
this.mapConfig = createVectorMapConfig([
{
id: 'wadus',
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
resolution: 'wadus',
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Invalid resolution, should be a number greather than 0' ],
errors_with_context:[{
type: 'layer',
message: 'Invalid resolution, should be a number greather than 0',
layer: {
"id": "wadus",
"index": 0,
"type": "mapnik"
}
}]
});
done();
});
});
it('should fail with bad placement', function (done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
placement: 'wadus',
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample'],
errors_with_context:[{
type: 'layer',
message: 'Invalid placement. Valid values: centroid, point-grid, point-sample',
layer: {
id: "layer0",
index: 0,
type: "mapnik",
}
}]
});
done();
});
});
it('should fail with bad threshold', function (done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 'wadus',
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Invalid threshold, should be a number greather than 0' ],
errors_with_context:[{
type: 'layer',
message: 'Invalid threshold, should be a number greather than 0',
layer: {
"id": "layer0",
"index": 0,
"type": "mapnik"
}
}]
});
done();
});
});
it('should fail with bad column name', function (done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
columns : {
'': {
aggregate_function: 'count',
aggregated_column: 'value',
}
},
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Invalid column name, should be a non empty string' ],
errors_with_context:[{
type: 'layer',
message: 'Invalid column name, should be a non empty string',
layer: {
"id": "layer0",
"index": 0,
"type": "mapnik"
}
}]
});
done();
});
});
it('should fail with bad aggregated function', function (done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
columns : {
'wadus_function': {
aggregate_function: 'wadus',
aggregated_column: 'value',
}
},
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Unsupported aggregation function wadus, ' +
'valid ones: count, avg, sum, min, max, mode' ],
errors_with_context:[{
type: 'layer',
message: 'Unsupported aggregation function wadus, ' +
'valid ones: count, avg, sum, min, max, mode',
layer: {
"id": "layer0",
"index": 0,
"type": "mapnik"
}
}]
});
done();
});
});
it('should fail with bad aggregated columns', function (done) {
this.mapConfig = createVectorMapConfig([
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
columns : {
'total_wadus': {
aggregate_function: 'sum',
aggregated_column: '',
}
},
}
}
}
]);
this.testClient = new TestClient(this.mapConfig);
const options = {
response: {
status: 400
}
};
this.testClient.getLayergroup(options, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, {
errors: [ 'Invalid aggregated column, should be a non empty string' ],
errors_with_context:[{
type: 'layer',
message: 'Invalid aggregated column, should be a non empty string',
layer: {
"id": "layer0",
"index": 0,
"type": "mapnik"
}
}]
});
done();
});
});
});
});
});

557
test/acceptance/dataviews/histogram.js
View File

@ -124,7 +124,7 @@ describe('histogram-dataview for date column type', function() {
"type": "cartodb",
"options": {
"source": {
"id": "datetime-histogram-source"
"id": "datetime-histogram-source-week"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
@ -134,7 +134,7 @@ describe('histogram-dataview for date column type', function() {
{
datetime_histogram: {
source: {
id: 'datetime-histogram-source'
id: 'datetime-histogram-source-week'
},
type: 'histogram',
options: {
@ -154,9 +154,109 @@ describe('histogram-dataview for date column type', function() {
offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds
}
},
datetime_histogram_automatic: {
datetime_histogram_automatic_second: {
source: {
id: 'datetime-histogram-source'
id: 'datetime-histogram-source-second'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_minute: {
source: {
id: 'datetime-histogram-source-minute'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_hour: {
source: {
id: 'datetime-histogram-source-hour'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_day: {
source: {
id: 'datetime-histogram-source-day'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_week: {
source: {
id: 'datetime-histogram-source-week'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_month: {
source: {
id: 'datetime-histogram-source-month'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_quarter: {
source: {
id: 'datetime-histogram-source-quarter'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_year: {
source: {
id: 'datetime-histogram-source-year'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_decade: {
source: {
id: 'datetime-histogram-source-decade'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_century: {
source: {
id: 'datetime-histogram-source-century'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
datetime_histogram_automatic_millennium: {
source: {
id: 'datetime-histogram-source-millennium'
},
type: 'histogram',
options: {
@ -197,13 +297,144 @@ describe('histogram-dataview for date column type', function() {
},
[
{
"id": "datetime-histogram-source",
"id": "datetime-histogram-source-second",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval",
"'2007-02-15 01:00:00'::timestamp, '2007-02-15 01:00:57'::timestamp,",
"'0.9 second'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-minute",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2007-02-15 02:00:57'::timestamp,",
"'75 second'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-hour",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2007-02-18 02:00:57'::timestamp,",
"'47 minutes'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-day",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2007-04-18 02:00:57'::timestamp,",
"'24 hours'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-week",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp,",
"'1 day'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-month",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2010-04-09 01:00:00'::timestamp,",
"'30 day'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-quarter",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2020-04-09 01:00:00'::timestamp,",
"'30 day'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-year",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'1990-02-15 01:00:00'::timestamp, '2018-04-09 01:00:00'::timestamp,",
"'30 day'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-decade",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'1850-02-15 01:00:00'::timestamp, '2018-04-09 01:00:00'::timestamp,",
"'30 day'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-century",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'0650-02-15 01:00:00'::timestamp, '1918-04-09 01:00:00'::timestamp,",
"'6 years'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-millennium",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'0005-02-15 01:00:00'::timestamp, '12000-04-09 01:00:00'::timestamp,",
"'72 years'::interval",
") date"
].join(' ')
}
@ -429,6 +660,47 @@ describe('histogram-dataview for date column type', function() {
});
});
it('should aggregate histogram using "second" aggregation ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS,
aggregation: 'second'
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_second', params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 57);
var initialTimestamp = '2007-02-15T01:00:00Z';
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function (bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.add(index, 'second')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
@ -470,6 +742,132 @@ describe('histogram-dataview for date column type', function() {
});
});
it('should aggregate histogram using "decade" aggregation ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS,
aggregation: 'decade'
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_decade', params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 17);
var initialTimestamp = '1850-01-01T00:00:00Z';
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function (bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.add(index * 10, 'year')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should aggregate histogram using "century" aggregation ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS,
aggregation: 'century'
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_century', params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 14);
var initialTimestamp = '0601-01-01T00:00:00Z';
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function (bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.add(index * 100, 'year')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should aggregate histogram using "millennium" aggregation ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS,
aggregation: 'millennium'
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_millennium', params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 12);
var initialTimestamp = '0001-01-01T00:00:00Z';
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function (bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.add(index * 1000, 'year')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var params = {
@ -533,19 +931,129 @@ describe('histogram-dataview for date column type', function() {
});
});
it('should find the best aggregation (automatic mode) to build the histogram', function (done) {
it('should find the best aggregation (automatic mode) to build the histogram: second', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) {
this.testClient.getDataview('datetime_histogram_automatic_second', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'week');
assert.equal(dataview.bins.length, 61);
assert.equal(dataview.bins_count, 61);
assert.equal(dataview.aggregation, 'second');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: minute', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_minute', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'minute');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: hour', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_hour', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'hour');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: day', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_day', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'day');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: week', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_week', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'week');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: month', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_month', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'month');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: quarter', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_quarter', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'quarter');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: year', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_year', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'year');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: decade', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_decade', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'decade');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: century', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_century', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'century');
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram: millennium', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic_millennium', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'millennium');
done();
});
});
it('should work with dates', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
@ -560,19 +1068,6 @@ describe('histogram-dataview for date column type', function() {
});
it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'week');
assert.equal(dataview.bins.length, 61);
assert.equal(dataview.bins_count, 61);
done();
});
});
it('should not apply offset for a histogram aggregated by minutes', function (done) {
var self = this;
var params = {
@ -858,13 +1353,15 @@ describe('histogram-dates: aggregation input value', function() {
assert.deepEqual(dataviewError, {
errors: [
'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year'
'Invalid aggregation value. Valid ones: auto, second, minute, ' +
'hour, day, week, month, quarter, year, decade, century, millennium'
],
errors_with_context: [{
type: 'unknown',
message: [
'Invalid aggregation value. ',
'Valid ones: auto, minute, hour, day, week, month, quarter, year'
'Valid ones: auto, second, minute, hour, day, week, month, ' +
'quarter, year, decade, century, millennium'
].join('')
}]
});
@ -887,13 +1384,15 @@ describe('histogram-dates: aggregation input value', function() {
assert.deepEqual(dataviewError, {
errors: [
'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year'
'Invalid aggregation value. Valid ones: auto, second, minute, ' +
'hour, day, week, month, quarter, year, decade, century, millennium'
],
errors_with_context: [{
type: 'unknown',
message: [
'Invalid aggregation value. ',
'Valid ones: auto, minute, hour, day, week, month, quarter, year'
'Valid ones: auto, second, minute, hour, day, week, month, ' +
'quarter, year, decade, century, millennium'
].join('')
}]
});
@ -966,7 +1465,7 @@ describe('histogram-dates: timestamp starts at epoch', function() {
const { aggregation, timestamp_start } = dataview;
assert.equal(timestamp_start, 0);
assert.equal(aggregation, 'month');
assert.equal(aggregation, 'quarter');
done();
});