cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP implemented date histogram

Browse Source
This commit is contained in:
Daniel García Aubert 2017-06-01 20:07:46 +02:00
parent 40a73f2eaf
commit 1d66e49910
5 changed files with 195 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/cartodb/backends/dataview.js
View File

@ -20,6 +20,8 @@ function DataviewBackend(analysisBackend) {
this.analysisBackend = analysisBackend;
}
var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'year'];
module.exports = DataviewBackend;
DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
@ -30,6 +32,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
mapConfigProvider.getMapConfig(this);
},
function runDataviewQuery(err, mapConfig) {
/* jshint maxcomplexity: 7 */
assert.ifError(err);
var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
@ -88,6 +91,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
{ownFilter: ownFilter}
);
if (params.aggregation && DATE_AGGREGATIONS.indexOf(params.aggregation) !== -1) {
overrideParams.aggregation = params.aggregation;
}
var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
dataview.getResult(pg, overrideParams, this);
},

1
lib/cartodb/controllers/base.js
View File

@ -25,6 +25,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
'start', // number
'end', // number
'column_type', // string
'aggregation', //string
// widgets search
'q'
];

96
lib/cartodb/models/dataview/histogram.js
View File

@ -8,7 +8,7 @@ dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
// var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
@ -71,7 +71,7 @@ var overrideBinsQueryTpl = dot.template([
var nullsQueryTpl = dot.template([
'nulls AS (',
' SELECT',
' count(*) AS nulls_count',
' count(*) AS nulls_count',
' FROM ({{=it._query}}) _cdb_histogram_nulls',
' WHERE {{=it._column}} IS NULL',
')'
@ -97,6 +97,52 @@ var histogramQueryTpl = dot.template([
'ORDER BY bin'
].join('\n'));
var dateBasicsQueryTpl = dot.template([
'basics AS (',
' SELECT',
' max(date_part(\'epoch\', {{=it._column}})) AS max_val,',
' min(date_part(\'epoch\', {{=it._column}})) AS min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,',
' min(date_trunc(\'{{=it._aggregation}}\', {{=it._column}})) AS start_date,',
' max({{=it._column}}) AS end_date,',
' count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
')'
].join(' \n'));
var dateBinsQueryTpl = dot.template([
'bins AS (',
' SELECT',
' bins_array,',
' ARRAY_LENGTH(bins_array, 1) AS bins_number',
' FROM (',
' SELECT',
' ARRAY(',
' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)',
' ) AS bins_array',
' FROM basics',
' ) _cdb_bins_array',
')'
].join('\n'));
var dateHistogramQueryTpl = dot.template([
'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,',
' nulls_count,',
' CASE WHEN min_val = max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1',
' END AS bin,',
' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
' count(*) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls',
'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
'ORDER BY bin'
].join('\n'));
var TYPE = 'histogram';
@ -118,6 +164,7 @@ function Histogram(query, options, queries) {
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
this.aggregation = options.aggregation;
this._columnType = null;
}
@ -163,7 +210,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
}
if (this._columnType === 'date') {
_column = columnCastTpl({column: _column});
return this._buildDateHistogramQuery(override, callback);
}
var _query = this.query;
@ -211,7 +258,6 @@ Histogram.prototype.sql = function(psql, override, callback) {
}
}
var histogramSql = [
"WITH",
[
@ -233,6 +279,48 @@ Histogram.prototype.sql = function(psql, override, callback) {
return callback(null, histogramSql);
};
Histogram.prototype._buildDateHistogramQuery = function (override, callback) {
var _column = this.column;
var _query = this.query;
var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
var dateBasicsQuery = dateBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation
});
var dateBinsQuery = [
dateBinsQueryTpl({
_aggregation: _aggregation
})
].join(',\n');
var nullsQuery = nullsQueryTpl({
_query: _query,
_column: _column
});
var dateHistogramQuery = dateHistogramQueryTpl({
_query: _query,
_column: _column
});
var histogramSql = [
"WITH",
[
dateBasicsQuery,
dateBinsQuery,
nullsQuery
].join(',\n'),
dateHistogramQuery
].join('\n');
debug(histogramSql);
return callback(null, histogramSql);
};
Histogram.prototype.format = function(result, override) {
override = override || {};
var buckets = [];

102
test/acceptance/dataviews/histogram.js
View File

@ -3,6 +3,15 @@ require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
describe('histogram-dataview', function() {
afterEach(function(done) {
@ -13,15 +22,6 @@ describe('histogram-dataview', function() {
}
});
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
var mapConfig = createMapConfig(
[
{
@ -94,6 +94,90 @@ describe('histogram-dataview', function() {
assert.equal(res.errors.length, 1);
assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/));
done();
});
});
});
describe('histogram-dataview for date column type', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "date-histogram-source"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
date_histogram: {
source: {
id: 'date-histogram-source'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'month'
}
}
},
[
{
"id": "date-histogram-source",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval",
") date"
].join(' ')
}
}
]
);
it('should create a date histogram aggregated in months', function (done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('date_histogram', {}, function(err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
dataview.bins.forEach(function(bin) {
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should override aggregation in weeks', function (done) {
var params = {
aggregation: 'week'
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('date_histogram', params, function(err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
dataview.bins.forEach(function(bin) {
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});

4
test/support/test-client.js
View File

@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
};
['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) {
['bbox', 'bins', 'start', 'end', 'aggregation'].forEach(function(extraParam) {
if (params.hasOwnProperty(extraParam)) {
urlParams[extraParam] = params[extraParam];
}
@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
params.placeholders = params.placeholders || {};
assert.response(server,
{
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),