From 1d66e49910376ceeb66efb044df651c8a52c4316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 1 Jun 2017 20:07:46 +0200 Subject: [PATCH] WIP implemented date histogram --- lib/cartodb/backends/dataview.js | 7 ++ lib/cartodb/controllers/base.js | 1 + lib/cartodb/models/dataview/histogram.js | 96 ++++++++++++++++++++- test/acceptance/dataviews/histogram.js | 102 +++++++++++++++++++++-- test/support/test-client.js | 4 +- 5 files changed, 195 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index e0515efe..84f0ccef 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,6 +20,8 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } +var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'year']; + module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { @@ -30,6 +32,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param mapConfigProvider.getMapConfig(this); }, function runDataviewQuery(err, mapConfig) { + /* jshint maxcomplexity: 7 */ assert.ifError(err); var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); @@ -88,6 +91,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {ownFilter: ownFilter} ); + if (params.aggregation && DATE_AGGREGATIONS.indexOf(params.aggregation) !== -1) { + overrideParams.aggregation = params.aggregation; + } + var dataview = dataviewFactory.getDataview(query, dataviewDefinition); dataview.getResult(pg, overrideParams, this); }, diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index ab6587c2..6ce49fa4 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -25,6 +25,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'start', // number 'end', // number 'column_type', // string + 'aggregation', //string // widgets search 'q' ]; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5d102bf5..6f3a2492 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -8,7 +8,7 @@ dot.templateSettings.strip = false; var columnTypeQueryTpl = dot.template( 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' ); -var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +// var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -71,7 +71,7 @@ var overrideBinsQueryTpl = dot.template([ var nullsQueryTpl = dot.template([ 'nulls AS (', ' SELECT', - ' count(*) AS nulls_count', + ' count(*) AS nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' @@ -97,6 +97,52 @@ var histogramQueryTpl = dot.template([ 'ORDER BY bin' ].join('\n')); +var dateBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' min(date_trunc(\'{{=it._aggregation}}\', {{=it._column}})) AS start_date,', + ' max({{=it._column}}) AS end_date,', + ' count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var dateBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT', + ' bins_array,', + ' ARRAY_LENGTH(bins_array, 1) AS bins_number', + ' FROM (', + ' SELECT', + ' ARRAY(', + ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' ) AS bins_array', + ' FROM basics', + ' ) _cdb_bins_array', + ')' +].join('\n')); + +var dateHistogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1', + ' END AS bin,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', + ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', + 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'ORDER BY bin' +].join('\n')); var TYPE = 'histogram'; @@ -118,6 +164,7 @@ function Histogram(query, options, queries) { this.queries = queries; this.column = options.column; this.bins = options.bins; + this.aggregation = options.aggregation; this._columnType = null; } @@ -163,7 +210,7 @@ Histogram.prototype.sql = function(psql, override, callback) { } if (this._columnType === 'date') { - _column = columnCastTpl({column: _column}); + return this._buildDateHistogramQuery(override, callback); } var _query = this.query; @@ -211,7 +258,6 @@ Histogram.prototype.sql = function(psql, override, callback) { } } - var histogramSql = [ "WITH", [ @@ -233,6 +279,48 @@ Histogram.prototype.sql = function(psql, override, callback) { return callback(null, histogramSql); }; +Histogram.prototype._buildDateHistogramQuery = function (override, callback) { + var _column = this.column; + var _query = this.query; + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + + var dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation + }); + + var dateBinsQuery = [ + dateBinsQueryTpl({ + _aggregation: _aggregation + }) + ].join(',\n'); + + var nullsQuery = nullsQueryTpl({ + _query: _query, + _column: _column + }); + + var dateHistogramQuery = dateHistogramQueryTpl({ + _query: _query, + _column: _column + }); + + var histogramSql = [ + "WITH", + [ + dateBasicsQuery, + dateBinsQuery, + nullsQuery + ].join(',\n'), + dateHistogramQuery + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index abbaef61..c581380a 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -3,6 +3,15 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + describe('histogram-dataview', function() { afterEach(function(done) { @@ -13,15 +22,6 @@ describe('histogram-dataview', function() { } }); - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - var mapConfig = createMapConfig( [ { @@ -94,6 +94,90 @@ describe('histogram-dataview', function() { assert.equal(res.errors.length, 1); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); + done(); + }); + }); + +}); + +describe('histogram-dataview for date column type', function() { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "date-histogram-source" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month' + } + } + }, + [ + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval", + ") date" + ].join(' ') + } + } + ] + ); + + it('should create a date histogram aggregated in months', function (done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override aggregation in weeks', function (done) { + var params = { + aggregation: 'week' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + done(); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 04980a5a..1913d018 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } @@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } params.placeholders = params.placeholders || {}; - + assert.response(server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),