From 660078f284caf65af724209f7c80b8b5dc48bfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 16:53:08 +0200 Subject: [PATCH 01/23] Fix minor issues with timezones --- NEWS.md | 6 + lib/cartodb/models/dataview/histogram.js | 4 +- test/acceptance/dataviews/histogram.js | 185 ++++++++++++++++++++++- 3 files changed, 192 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9ca6d4a4..3d5d5722 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,12 @@ ## 3.10.2 Released 2017-mm-dd +Announcements: + - Allow to override with any aggregation for histograms instantiated w/o aggregation. + +Bug fixes: + - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + ## 3.10.1 Released 2017-08-04 diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 76b42c03..891cc2ec 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -246,7 +246,7 @@ var dateHistogramQueryTpl = dot.template([ ' date_part(', ' \'epoch\', ', ' date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', @@ -333,7 +333,7 @@ Histogram.prototype._buildQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; - if (this._columnType === 'date' && this.aggregation !== undefined) { + if (this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined)) { return this._buildDateHistogramQuery(psql, override, callback); } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index a75cf2b1..84e2c15e 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -888,7 +888,6 @@ describe('histogram-dates: aggregation input value', function() { }); }); - describe('histogram-dates: timestamp starts at epoch', function() { afterEach(function(done) { @@ -958,3 +957,187 @@ describe('histogram-dates: timestamp starts at epoch', function() { }); }); }); + +describe('histogram-dates: trunc timestamp for each bin respecting user\'s timezone', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should return histogram with two buckets', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); +}); + + +describe('histogram: be able to override with aggregation for histograms instantiated w/o aggregation', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should apply aggregation to the histogram', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); +}); From 1f8da14c2ae6dff1e3353d04ac821f89cfe36430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 18:27:24 +0200 Subject: [PATCH 02/23] Cast to timestamp start_date and end_date to calculate bins when date column is timestamptz --- lib/cartodb/models/dataview/histogram.js | 4 +- test/acceptance/dataviews/histogram.js | 88 ++++++++++++++++-------- 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 891cc2ec..a044bfc7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -180,9 +180,9 @@ var dateBasicsQueryTpl = dot.template([ ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._offset}}\'', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )) AS start_date,', - ' max({{=it._column}} AT TIME ZONE \'{{=it._offset}}\') AS end_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 84e2c15e..3baf002b 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -991,6 +991,16 @@ describe('histogram-dates: trunc timestamp for each bin respecting user\'s timez column: 'd', aggregation: 'auto' } + }, + timezone_epoch_histogram_tz: { + source: { + id: 'a1' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } } }, [ @@ -1007,45 +1017,69 @@ describe('histogram-dates: trunc timestamp for each bin respecting user\'s timez ') date' ].join(' ') } + }, + { + id: 'a1', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamptz,', + '\'1970-01-01 01:59:00\'::timestamptz,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } } ] ); - it('should return histogram with two buckets', function(done) { - this.testClient = new TestClient(mapConfig, 1234); + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'timezone_epoch_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'timezone_epoch_histogram' + }]; - const override = { - aggregation: 'day', - offset: '-3600' - }; + dateHistogramsUseCases.forEach(function (test) { + it('should return histogram with two buckets ' + test.desc , function(done) { + this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { - assert.ifError(err); + const override = { + aggregation: 'day', + offset: '-3600' + }; - var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 - var initialTimestamp = '1969-12-31T00:00:00-01:00'; - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(OFFSET_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); + this.testClient.getDataview(test.dataviewId, override, function(err, dataview) { + assert.ifError(err); - dataview.bins.forEach(function (bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(OFFSET_IN_MINUTES) - .add(index, override.aggregation) - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(OFFSET_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); }); }); From 2024e89c6ab227091e680d5cf74c480ccaf1795d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 18:30:17 +0200 Subject: [PATCH 03/23] Update news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 3d5d5722..ced38066 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,7 @@ Announcements: Bug fixes: - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + - Support timestamp with timezones to calculate the number of bins in time-series. ## 3.10.1 From b2bbc329ea9a542925644f5e3a6b540ffb6e2fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:03:49 +0200 Subject: [PATCH 04/23] Apply prefix for intermediate query variables to avoid name colision --- lib/cartodb/models/dataview/histogram.js | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index a044bfc7..6d399d39 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -176,14 +176,14 @@ var histogramQueryTpl = dot.template([ var dateBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', - ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', - ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(date_trunc(', ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )) AS start_date,', - ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS end_date,', - ' count(1) AS total_rows', + ' )) AS __cdb_start_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' ].join(' \n')); @@ -191,19 +191,19 @@ var dateBasicsQueryTpl = dot.template([ var dateOverrideBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val,', - ' min({{=it._start}}) AS min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' max({{=it._end}}) AS __cdb_max_val,', + ' min({{=it._start}}) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', ' min(', ' date_trunc(', ' \'{{=it._aggregation}}\',', ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )', - ' ) AS start_date,', + ' ) AS __cdb_start_date,', ' max(', ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AS end_date,', - ' count(1) AS total_rows', + ' ) AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' ].join(' \n')); @@ -217,8 +217,8 @@ var dateBinsQueryTpl = dot.template([ ' SELECT', ' ARRAY(', ' SELECT GENERATE_SERIES(', - ' start_date::timestamptz,', - ' end_date::timestamptz,', + ' __cdb_start_date::timestamptz,', + ' __cdb_end_date::timestamptz,', ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', ' ) AS bins_array', @@ -229,10 +229,10 @@ var dateBinsQueryTpl = dot.template([ var dateHistogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' (__cdb_max_val - __cdb_min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', - ' CASE WHEN min_val = max_val', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', @@ -250,14 +250,14 @@ var dateHistogramQueryTpl = dot.template([ ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', - ' date_part(\'epoch\', start_date)::numeric AS timestamp_start,', + ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, __cdb_start_date', 'ORDER BY bin' ].join('\n')); From 0f45675652a87488ba261e0c1035fed650d6ed78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:06:31 +0200 Subject: [PATCH 05/23] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index ced38066..bd89d748 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ Announcements: Bug fixes: - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. - Support timestamp with timezones to calculate the number of bins in time-series. + - Fixed issue related to name collision while building time-series query. ## 3.10.1 From 5c80ff81917f4093a1ac9c3e3dc3964b4b17337a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Aug 2017 19:24:15 +0200 Subject: [PATCH 06/23] Extract query: move condition to a method --- lib/cartodb/models/dataview/histogram.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6d399d39..4af16d65 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -328,12 +328,16 @@ Histogram.prototype.sql = function(psql, override, callback) { this._buildQuery(psql, override, callback); }; +Histogram.prototype.isDateHistogram = function (override) { + return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); +}; + Histogram.prototype._buildQuery = function (psql, override, callback) { var filteredQuery, basicsQuery, binsQuery; var _column = this.column; var _query = this.query; - if (this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined)) { + if (this.isDateHistogram(override)) { return this._buildDateHistogramQuery(psql, override, callback); } From 994e58bef7e2f9f63a66b6c04e4bf9d38428915f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:33:26 +0200 Subject: [PATCH 07/23] Add prefix to bins_number and nulls_cout to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4af16d65..dce22bb0 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -109,7 +109,7 @@ var binsQueryTpl = dot.template([ ' {{=it._maxBins}}', ' )', ' )', - ' END AS bins_number', + ' END AS __cdb_bins_number', ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' @@ -117,14 +117,14 @@ var binsQueryTpl = dot.template([ var overrideBinsQueryTpl = dot.template([ 'bins AS (', - ' SELECT {{=it._bins}} AS bins_number', + ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); var nullsQueryTpl = dot.template([ 'nulls AS (', ' SELECT', - ' count(*) AS nulls_count', + ' count(*) AS __cdb_nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' @@ -153,22 +153,22 @@ var nansQueryTpl = dot.template([ var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', + ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', ' {{?it._isFloatColumn}}infinities_count,', ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number), __cdb_bins_number)) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', - 'GROUP BY bin, bins_number, bin_width, nulls_count,', + 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -212,7 +212,7 @@ var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', ' bins_array,', - ' ARRAY_LENGTH(bins_array, 1) AS bins_number', + ' ARRAY_LENGTH(bins_array, 1) AS __cdb_bins_number', ' FROM (', ' SELECT', ' ARRAY(', @@ -229,9 +229,9 @@ var dateBinsQueryTpl = dot.template([ var dateHistogramQueryTpl = dot.template([ 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', @@ -239,7 +239,7 @@ var dateHistogramQueryTpl = dot.template([ ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', ' bins_array', ' ),', - ' bins_number', + ' __cdb_bins_number', ' )) - 1', ' END AS bin,', ' min(', @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, __cdb_start_date', + 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count, __cdb_avg_val, __cdb_start_date', 'ORDER BY bin' ].join('\n')); From 823925d09143b75e96c1f9d308bba741ce7ecf4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:41:22 +0200 Subject: [PATCH 08/23] Add prefix to bins_array to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dce22bb0..c6f4ffbb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -211,8 +211,8 @@ var dateOverrideBasicsQueryTpl = dot.template([ var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', - ' bins_array,', - ' ARRAY_LENGTH(bins_array, 1) AS __cdb_bins_number', + ' __cdb_bins_array,', + ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', ' FROM (', ' SELECT', ' ARRAY(', @@ -221,9 +221,9 @@ var dateBinsQueryTpl = dot.template([ ' __cdb_end_date::timestamptz,', ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', - ' ) AS bins_array', + ' ) AS __cdb_bins_array', ' FROM basics', - ' ) _cdb_bins_array', + ' ) _cdb_bins_array_query', ')' ].join('\n')); @@ -237,7 +237,7 @@ var dateHistogramQueryTpl = dot.template([ ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', - ' bins_array', + ' __cdb_bins_array', ' ),', ' __cdb_bins_number', ' )) - 1', From e4461601512b40f599e62e97221d6c8690d334ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:45:05 +0200 Subject: [PATCH 09/23] Use final columns to group by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c6f4ffbb..d868513a 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count, __cdb_avg_val, __cdb_start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, timestamp_start', 'ORDER BY bin' ].join('\n')); From 91d5a0e4e4c38540606ac5bf322c02a2fe360c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 08:46:26 +0200 Subject: [PATCH 10/23] Remove column avg_val in group_by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index d868513a..46f86a5d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -257,7 +257,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, __cdb_avg_val, timestamp_start', + 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' ].join('\n')); From 0f8de9e74b920d0d8ab20ce379bf930c44011756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:06:36 +0200 Subject: [PATCH 11/23] Add prefix to basics query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 28 ++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 46f86a5d..c9204960 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -67,7 +67,7 @@ var filteredQueryTpl = dot.template([ ].join(' \n')); var basicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', @@ -76,7 +76,7 @@ var basicsQueryTpl = dot.template([ ].join(' \n')); var overrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', @@ -110,7 +110,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS __cdb_bins_number', - ' FROM basics, iqrange, filtered_source', + ' FROM __cdb_basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -161,20 +161,26 @@ var histogramQueryTpl = dot.template([ ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number), __cdb_bins_number)) - 1', + ' ELSE GREATEST(', + ' 1,', + ' LEAST(', + ' WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number),', + ' __cdb_bins_number', + ' )', + ' ) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); var dateBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', @@ -184,12 +190,12 @@ var dateBasicsQueryTpl = dot.template([ ' )) AS __cdb_start_date,', ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM ({{=it._query}}) __cdb_basics_query', ')' ].join(' \n')); var dateOverrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', ' max({{=it._end}}) AS __cdb_max_val,', ' min({{=it._start}}) AS __cdb_min_val,', @@ -204,7 +210,7 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AS __cdb_end_date,', ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM ({{=it._query}}) __cdb_basics_query', ')' ].join(' \n')); @@ -222,7 +228,7 @@ var dateBinsQueryTpl = dot.template([ ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', ' ) AS __cdb_bins_array', - ' FROM basics', + ' FROM __cdb_basics', ' ) _cdb_bins_array_query', ')' ].join('\n')); @@ -255,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From f3ababffc1f29e8f2fb8e5fed34b158888e56429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:12:52 +0200 Subject: [PATCH 12/23] Prefix bins query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c9204960..2af66449 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -99,7 +99,7 @@ var iqrQueryTpl = dot.template([ ].join('\n')); var binsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', ' THEN 1', ' ELSE GREATEST(', @@ -116,7 +116,7 @@ var binsQueryTpl = dot.template([ ].join('\n')); var overrideBinsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' @@ -215,7 +215,7 @@ var dateOverrideBasicsQueryTpl = dot.template([ ].join(' \n')); var dateBinsQueryTpl = dot.template([ - 'bins AS (', + '__cdb_bins AS (', ' SELECT', ' __cdb_bins_array,', ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', @@ -261,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From 3ee064a59fa87cb2d0802568af1fd8a9d190d465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:32:13 +0200 Subject: [PATCH 13/23] Prefix nulls query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 2af66449..54a9fcf6 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -122,7 +122,7 @@ var overrideBinsQueryTpl = dot.template([ ].join('\n')); var nullsQueryTpl = dot.template([ - 'nulls AS (', + '__cdb_nulls AS (', ' SELECT', ' count(*) AS __cdb_nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' @@ -261,7 +261,7 @@ var dateHistogramQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, nulls', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', 'ORDER BY bin' From a9e3bc3cdadfbfd1d57cf609442096499f1100c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:36:23 +0200 Subject: [PATCH 14/23] Prefix filtered source to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 54a9fcf6..629a69f7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -52,9 +52,9 @@ var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; var filteredQueryTpl = dot.template([ - 'filtered_source AS (', + '__cdb_filtered_source AS (', ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source', + ' FROM ({{=it._query}}) _cdb_filtered_source_query', ' WHERE', ' {{=it._column}} IS NOT NULL', ' {{?it._isFloatColumn}}AND', @@ -71,7 +71,7 @@ var basicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' FROM __cdb_filtered_source', ')' ].join(' \n')); @@ -80,7 +80,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' FROM __cdb_filtered_source', ')' ].join('\n')); @@ -91,7 +91,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM filtered_source) _cdb_quartiles', + ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -110,7 +110,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS __cdb_bins_number', - ' FROM __cdb_basics, iqrange, filtered_source', + ' FROM __cdb_basics, iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From 7e058955ea71ac88010855c3d3d03fc56226e22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:39:51 +0200 Subject: [PATCH 15/23] Use final naming to group by --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 629a69f7..7099340d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -174,7 +174,7 @@ var histogramQueryTpl = dot.template([ ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', - 'GROUP BY bin, __cdb_bins_number, bin_width, __cdb_nulls_count,', + 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); From 34cf45bc9dedbfbc4595ac1e7a8151b606e90716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:43:31 +0200 Subject: [PATCH 16/23] Prefix infinities query to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7099340d..0edfaae9 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -131,7 +131,7 @@ var nullsQueryTpl = dot.template([ ].join('\n')); var infinitiesQueryTpl = dot.template([ - 'infinities AS (', + '__cdb_infinities AS (', ' SELECT', ' count(*) AS infinities_count', ' FROM ({{=it._query}}) _cdb_histogram_infinities', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From c7e5dbf1585a541fe2b5328f576896770ecc691d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:46:34 +0200 Subject: [PATCH 17/23] Fix query aliases --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 0edfaae9..01335421 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -134,7 +134,7 @@ var infinitiesQueryTpl = dot.template([ '__cdb_infinities AS (', ' SELECT', ' count(*) AS infinities_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' FROM ({{=it._query}}) __cdb_infinities_query', ' WHERE', ' {{=it._column}} = \'infinity\'::float', ' OR', @@ -146,7 +146,7 @@ var nansQueryTpl = dot.template([ 'nans AS (', ' SELECT', ' count(*) AS nans_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' FROM ({{=it._query}}) __cdb_nans_query', ' WHERE {{=it._column}} = \'NaN\'::float', ')' ].join('\n')); From cf71489a7f3c18718e0aefdd5c5b84997bc26960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:54:49 +0200 Subject: [PATCH 18/23] Prefix nans and infinities counters to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 01335421..26ee452c 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -133,7 +133,7 @@ var nullsQueryTpl = dot.template([ var infinitiesQueryTpl = dot.template([ '__cdb_infinities AS (', ' SELECT', - ' count(*) AS infinities_count', + ' count(*) AS __cdb_infinities_count', ' FROM ({{=it._query}}) __cdb_infinities_query', ' WHERE', ' {{=it._column}} = \'infinity\'::float', @@ -143,9 +143,9 @@ var infinitiesQueryTpl = dot.template([ ].join('\n')); var nansQueryTpl = dot.template([ - 'nans AS (', + '__cdb_nans AS (', ' SELECT', - ' count(*) AS nans_count', + ' count(*) AS __cdb_nans_count', ' FROM ({{=it._query}}) __cdb_nans_query', ' WHERE {{=it._column}} = \'NaN\'::float', ')' @@ -156,8 +156,8 @@ var histogramQueryTpl = dot.template([ ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', ' __cdb_bins_number AS bins_number,', ' __cdb_nulls_count AS nulls_count,', - ' {{?it._isFloatColumn}}infinities_count,', - ' nans_count,{{?}}', + ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', + ' __cdb_nans_count AS nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -173,7 +173,7 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From ff3d7ed7b21829194518a2d75a84c80d3d4bc126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 09:57:12 +0200 Subject: [PATCH 19/23] Fix jshint typo --- lib/cartodb/models/dataview/histogram.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 26ee452c..ec3e616e 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -173,7 +173,8 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', + ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' From 7e159c565b61ffdb2bfdbe3e385b5d6daae8541d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:03:40 +0200 Subject: [PATCH 20/23] Prefix iqr query calculation to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index ec3e616e..9c63b836 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -85,8 +85,8 @@ var overrideBasicsQueryTpl = dot.template([ ].join('\n')); var iqrQueryTpl = dot.template([ - 'iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + '__cdb_iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', ' FROM (', ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', @@ -100,17 +100,17 @@ var iqrQueryTpl = dot.template([ var binsQueryTpl = dot.template([ '__cdb_bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + ' SELECT CASE WHEN total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((max_val - min_val) / (2 * __cdb_iqr * power(total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', ' END AS __cdb_bins_number', - ' FROM __cdb_basics, iqrange, __cdb_filtered_source', + ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); From 3c061769c68582eb645a8e6184bbd3cedd13046e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:11:35 +0200 Subject: [PATCH 21/23] Prefix basics columns to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 9c63b836..7d29d01d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -69,8 +69,8 @@ var filteredQueryTpl = dot.template([ var basicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', ' FROM __cdb_filtered_source', ')' ].join(' \n')); @@ -78,8 +78,8 @@ var basicsQueryTpl = dot.template([ var overrideBasicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', ' FROM __cdb_filtered_source', ')' ].join('\n')); @@ -100,12 +100,12 @@ var iqrQueryTpl = dot.template([ var binsQueryTpl = dot.template([ '__cdb_bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR __cdb_iqr = 0', + ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * __cdb_iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', @@ -153,18 +153,18 @@ var nansQueryTpl = dot.template([ var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', ' __cdb_bins_number AS bins_number,', ' __cdb_nulls_count AS nulls_count,', ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', ' __cdb_nans_count AS nans_count,{{?}}', - ' avg_val,', - ' CASE WHEN min_val = max_val', + ' __cdb_avg_val AS avg_val,', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', ' ELSE GREATEST(', ' 1,', ' LEAST(', - ' WIDTH_BUCKET({{=it._column}}, min_val, max_val, __cdb_bins_number),', + ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', ' __cdb_bins_number', ' )', ' ) - 1', From e7a0b246a39cd722138196c68b657b94bd1154e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:20:36 +0200 Subject: [PATCH 22/23] Prefix with double underscore --- lib/cartodb/models/dataview/histogram.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7d29d01d..b07aa474 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -54,7 +54,7 @@ var BIN_MAX_NUMBER = 48; var filteredQueryTpl = dot.template([ '__cdb_filtered_source AS (', ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source_query', + ' FROM ({{=it._query}}) __cdb_filtered_source_query', ' WHERE', ' {{=it._column}} IS NOT NULL', ' {{?it._isFloatColumn}}AND', @@ -94,7 +94,7 @@ var iqrQueryTpl = dot.template([ ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', - ' ) _cdb_iqr', + ' ) __cdb_iqr', ')' ].join('\n')); @@ -125,7 +125,7 @@ var nullsQueryTpl = dot.template([ '__cdb_nulls AS (', ' SELECT', ' count(*) AS __cdb_nulls_count', - ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' FROM ({{=it._query}}) __cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' ].join('\n')); @@ -230,7 +230,7 @@ var dateBinsQueryTpl = dot.template([ ' )', ' ) AS __cdb_bins_array', ' FROM __cdb_basics', - ' ) _cdb_bins_array_query', + ' ) __cdb_bins_array_query', ')' ].join('\n')); From c643160671b5a40dec233192d33a6aaafc0667e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 8 Aug 2017 10:32:53 +0200 Subject: [PATCH 23/23] Prefix date interval query (to calculate automatic aggregation) to avoid name collision --- lib/cartodb/models/dataview/histogram.js | 50 ++++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b07aa474..c27c696f 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -9,42 +9,42 @@ var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var dateIntervalQueryTpl = dot.template([ 'WITH', - 'dates AS (', + '__cdb_dates AS (', ' SELECT', - ' MAX({{=it.column}}::timestamp) AS _end,', - ' MIN({{=it.column}}::timestamp) AS _start', - ' FROM ({{=it.query}}) _cdb_source', + ' MAX({{=it.column}}::timestamp) AS __cdb_end,', + ' MIN({{=it.column}}::timestamp) AS __cdb_start', + ' FROM ({{=it.query}}) __cdb_source', '),', - 'interval_in_days AS (', + '__cdb_interval_in_days AS (', ' SELECT' , - ' DATE_PART(\'day\', _end - _start) AS days', - ' FROM dates', + ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', + ' FROM __cdb_dates', '),', - 'interval_in_hours AS (', + '__cdb_interval_in_hours AS (', ' SELECT', - ' days * 24 + DATE_PART(\'hour\', _end - _start) AS hours', - ' FROM interval_in_days, dates', + ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', + ' FROM __cdb_interval_in_days, __cdb_dates', '),', - 'interval_in_minutes AS (', + '__cdb_interval_in_minutes AS (', ' SELECT', - ' hours * 60 + DATE_PART(\'minute\', _end - _start) AS minutes', - ' FROM interval_in_hours, dates', + ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', + ' FROM __cdb_interval_in_hours, __cdb_dates', '),', - 'interval_in_seconds AS (', + '__cdb_interval_in_seconds AS (', ' SELECT', - ' minutes * 60 + DATE_PART(\'second\', _end - _start) AS seconds', - ' FROM interval_in_minutes, dates', + ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', + ' FROM __cdb_interval_in_minutes, __cdb_dates', ')', 'SELECT', - ' ROUND(days / 365) AS year,', - ' ROUND(days / 90) AS quarter,', - ' ROUND(days / 30) AS month,', - ' ROUND(days / 7) AS week,', - ' days AS day,', - ' hours AS hour,', - ' minutes AS minute,', - ' seconds AS second', - 'FROM interval_in_days, interval_in_hours, interval_in_minutes, interval_in_seconds' + ' ROUND(__cdb_days / 365) AS year,', + ' ROUND(__cdb_days / 90) AS quarter,', + ' ROUND(__cdb_days / 30) AS month,', + ' ROUND(__cdb_days / 7) AS week,', + ' __cdb_days AS day,', + ' __cdb_hours AS hour,', + ' __cdb_minutes AS minute,', + ' __cdb_seconds AS second', + 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' ].join('\n')); var MAX_INTERVAL_VALUE = 366;