WIP implemented date histogram

lib/cartodb/backends/dataview.js

@@ -20,6 +20,8 @@ function DataviewBackend(analysisBackend) {
     this.analysisBackend = analysisBackend;
 }
 
+var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'year'];
+
 module.exports = DataviewBackend;
 
 DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
@@ -30,6 +32,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             mapConfigProvider.getMapConfig(this);
         },
         function runDataviewQuery(err, mapConfig) {
+            /* jshint maxcomplexity: 7 */
             assert.ifError(err);
 
             var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
@@ -88,6 +91,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
                 {ownFilter: ownFilter}
             );
 
+            if (params.aggregation && DATE_AGGREGATIONS.indexOf(params.aggregation) !== -1) {
+                overrideParams.aggregation = params.aggregation;
+            }
+
             var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
             dataview.getResult(pg, overrideParams, this);
         },

lib/cartodb/controllers/base.js

@@ -25,6 +25,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
     'start', // number
     'end', // number
     'column_type', // string
+    'aggregation', //string
     // widgets search
     'q'
 ];

lib/cartodb/models/dataview/histogram.js

@@ -8,7 +8,7 @@ dot.templateSettings.strip = false;
 var columnTypeQueryTpl = dot.template(
     'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
 );
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+// var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
 
 var BIN_MIN_NUMBER = 6;
 var BIN_MAX_NUMBER = 48;
@@ -97,6 +97,52 @@ var histogramQueryTpl = dot.template([
     'ORDER BY bin'
 ].join('\n'));
 
+var dateBasicsQueryTpl = dot.template([
+    'basics AS (',
+    '    SELECT',
+    '        max(date_part(\'epoch\', {{=it._column}})) AS max_val,',
+    '        min(date_part(\'epoch\', {{=it._column}})) AS min_val,',
+    '        avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,',
+    '        min(date_trunc(\'{{=it._aggregation}}\', {{=it._column}})) AS start_date,',
+    '        max({{=it._column}}) AS end_date,',
+    '        count(1) AS total_rows',
+    '    FROM ({{=it._query}}) _cdb_basics',
+    ')'
+].join(' \n'));
+
+var dateBinsQueryTpl = dot.template([
+    'bins AS (',
+    '    SELECT',
+    '        bins_array,',
+    '        ARRAY_LENGTH(bins_array, 1) AS bins_number',
+    '    FROM (',
+    '       SELECT',
+    '           ARRAY(',
+    '               SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)',
+    '           ) AS bins_array',
+    '       FROM basics',
+    '    ) _cdb_bins_array',
+    ')'
+].join('\n'));
+
+var dateHistogramQueryTpl = dot.template([
+    'SELECT',
+    '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
+    '    bins_number,',
+    '    nulls_count,',
+    '    CASE WHEN min_val = max_val',
+    '      THEN 0',
+    '      ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1',
+    '    END AS bin,',
+    '    min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
+    '    max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
+    '    avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
+    '    count(*) AS freq',
+    'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls',
+    'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
+    'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
+    'ORDER BY bin'
+].join('\n'));
 
 var TYPE = 'histogram';
 
@@ -118,6 +164,7 @@ function Histogram(query, options, queries) {
     this.queries = queries;
     this.column = options.column;
     this.bins = options.bins;
+    this.aggregation = options.aggregation;
 
     this._columnType = null;
 }
@@ -163,7 +210,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
     }
 
     if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
+        return this._buildDateHistogramQuery(override, callback);
     }
 
     var _query = this.query;
@@ -211,7 +258,6 @@ Histogram.prototype.sql = function(psql, override, callback) {
         }
     }
 
-
     var histogramSql = [
         "WITH",
         [
@@ -233,6 +279,48 @@ Histogram.prototype.sql = function(psql, override, callback) {
     return callback(null, histogramSql);
 };
 
+Histogram.prototype._buildDateHistogramQuery = function (override, callback) {
+    var _column = this.column;
+    var _query = this.query;
+    var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
+
+    var dateBasicsQuery = dateBasicsQueryTpl({
+        _query: _query,
+        _column: _column,
+        _aggregation: _aggregation
+    });
+
+    var dateBinsQuery = [
+        dateBinsQueryTpl({
+            _aggregation: _aggregation
+        })
+    ].join(',\n');
+
+    var nullsQuery = nullsQueryTpl({
+        _query: _query,
+        _column: _column
+    });
+
+    var dateHistogramQuery = dateHistogramQueryTpl({
+        _query: _query,
+        _column: _column
+    });
+
+    var histogramSql = [
+        "WITH",
+        [
+            dateBasicsQuery,
+            dateBinsQuery,
+            nullsQuery
+        ].join(',\n'),
+        dateHistogramQuery
+    ].join('\n');
+
+    debug(histogramSql);
+
+    return callback(null, histogramSql);
+};
+
 Histogram.prototype.format = function(result, override) {
     override = override || {};
     var buckets = [];

test/acceptance/dataviews/histogram.js

@@ -3,6 +3,15 @@ require('../../support/test_helper');
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
+function createMapConfig(layers, dataviews, analysis) {
+    return {
+        version: '1.5.0',
+        layers: layers,
+        dataviews: dataviews || {},
+        analyses: analysis || []
+    };
+}
+
 describe('histogram-dataview', function() {
 
     afterEach(function(done) {
@@ -13,15 +22,6 @@ describe('histogram-dataview', function() {
         }
     });
 
-    function createMapConfig(layers, dataviews, analysis) {
-        return {
-            version: '1.5.0',
-            layers: layers,
-            dataviews: dataviews || {},
-            analyses: analysis || []
-        };
-    }
-
     var mapConfig = createMapConfig(
         [
             {
@@ -94,6 +94,90 @@ describe('histogram-dataview', function() {
             assert.equal(res.errors.length, 1);
             assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/));
 
+            done();
+        });
+    });
+
+});
+
+describe('histogram-dataview for date column type', function() {
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "date-histogram-source"
+                    },
+                    "cartocss": "#points { marker-width: 10; marker-fill: red; }",
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            date_histogram: {
+                source: {
+                    id: 'date-histogram-source'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'month'
+                }
+            }
+        },
+        [
+            {
+                "id": "date-histogram-source",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            }
+        ]
+    );
+
+    it('should create a date histogram aggregated in months', function (done) {
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('date_histogram', {}, function(err, dataview) {
+            assert.ok(!err, err);
+            assert.equal(dataview.type, 'histogram');
+            assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
+            dataview.bins.forEach(function(bin) {
+                assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
+            });
+
+            done();
+        });
+    });
+
+    it('should override aggregation in weeks', function (done) {
+        var params = {
+            aggregation: 'week'
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('date_histogram', params, function(err, dataview) {
+            assert.ok(!err, err);
+            assert.equal(dataview.type, 'histogram');
+            assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
+            dataview.bins.forEach(function(bin) {
+                assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
+            });
+
             done();
         });
     });

test/support/test-client.js

@@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
                 own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
             };
 
-            ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) {
+            ['bbox', 'bins', 'start', 'end', 'aggregation'].forEach(function(extraParam) {
                 if (params.hasOwnProperty(extraParam)) {
                     urlParams[extraParam] = params[extraParam];
                 }