From 6c301403e37e4f1901c5d43d54e5928e2de20d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 15:59:33 +0200 Subject: [PATCH 01/60] Histogram going red: fails while quering Infinity and NanN values --- test/acceptance/dataviews/histogram.js | 86 +++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 10 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index abbaef61..c75e9baf 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -3,6 +3,15 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + describe('histogram-dataview', function() { afterEach(function(done) { @@ -13,15 +22,6 @@ describe('histogram-dataview', function() { } }); - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - var mapConfig = createMapConfig( [ { @@ -89,7 +89,6 @@ describe('histogram-dataview', function() { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('pop_max_histogram', params, function(err, res) { assert.ok(!err, err); - assert.ok(res.errors); assert.equal(res.errors.length, 1); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); @@ -98,3 +97,70 @@ describe('histogram-dataview', function() { }); }); }); + + +describe('histogram-dataview: special float valuer', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_histogram', {}, function(err, dataview) { + assert.ok(!err, err); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); From cd53eda0a5062551fb7f64efaa11c82acd70bec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 16:01:41 +0200 Subject: [PATCH 02/60] Handle Infinity and NaN values for histograms --- lib/cartodb/models/dataview/histogram.js | 75 +++++++++++++++++++++--- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5d102bf5..f6b2635d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -13,12 +13,27 @@ var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join(' \n')); @@ -27,7 +42,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join('\n')); @@ -38,7 +53,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -57,7 +72,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -77,11 +92,34 @@ var nullsQueryTpl = dot.template([ ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + 'infinities AS (', + ' SELECT', + ' count(*) AS infinities_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + 'nans AS (', + ' SELECT', + ' count(*) AS nans_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', + ' infinities_count,', + ' nans_count,', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -91,9 +129,8 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'FROM filtered_source, basics, nulls, infinities, nans, bins', + 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', 'ORDER BY bin' ].join('\n')); @@ -168,7 +205,12 @@ Histogram.prototype.sql = function(psql, override, callback) { var _query = this.query; - var basicsQuery, binsQuery; + var filteredQuery, basicsQuery, binsQuery; + + filteredQuery = filteredQueryTpl({ + _query: _query, + _column: _column + }); if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { debug('overriding with %j', override); @@ -215,11 +257,20 @@ Histogram.prototype.sql = function(psql, override, callback) { var histogramSql = [ "WITH", [ + filteredQuery, basicsQuery, binsQuery, nullsQueryTpl({ _query: _query, _column: _column + }), + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ + _query: _query, + _column: _column }) ].join(',\n'), histogramQueryTpl({ @@ -241,6 +292,8 @@ Histogram.prototype.format = function(result, override) { var width = getWidth(override); var binsStart = getBinStart(override); var nulls = 0; + var infinities = 0; + var nans = 0; var avg; if (result.rows.length) { @@ -249,10 +302,12 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; + infinities = firstRow.infinities_count; + nans = firstRow.nans_count; binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'infinities_count', 'nans_count', 'avg_val'); }); } @@ -261,6 +316,8 @@ Histogram.prototype.format = function(result, override) { bins_count: binsCount, bins_start: binsStart, nulls: nulls, + infinities: infinities, + nans: nans, avg: avg, bins: buckets }; From 7a3498e8ec2553629f89b0d2298386905e4b2bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 9 Jun 2017 12:17:16 +0200 Subject: [PATCH 03/60] Going red: formula does not work with Infinity or NaN values --- test/acceptance/dataviews/formula.js | 80 ++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/acceptance/dataviews/formula.js diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js new file mode 100644 index 00000000..93db6b38 --- /dev/null +++ b/test/acceptance/dataviews/formula.js @@ -0,0 +1,80 @@ +require('../../support/test_helper'); +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + +describe('formula-dataview: special float valuer', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_formula: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: 'val', + operation: 'avg' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_formula', {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.result, 501) + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); From 9ba65bd5a482e08a9141148f270ef7636cd8580d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 9 Jun 2017 15:18:37 +0200 Subject: [PATCH 04/60] Going green: Fix test for formula overviews flavour --- lib/cartodb/models/dataview/formula.js | 24 ++++++++++++++--- test/acceptance/dataviews/overviews.js | 36 +++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 156985c5..1421032d 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -7,9 +7,21 @@ dot.templateSettings.strip = false; var formulaQueryTpl = dot.template([ 'SELECT', - '{{=it._operation}}({{=it._column}}) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + ' {{=it._operation}}({{=it._column}}) AS result,', + ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + ' {{?it._operation!==\'count\'}}', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + ' {{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._operation!==\'count\'}}', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float', + '{{?}}' ].join('\n')); var VALID_OPERATIONS = { @@ -78,13 +90,17 @@ Formula.prototype.format = function(result) { var formattedResult = { operation: this.operation, result: 0, - nulls: 0 + nulls: 0, + nans: 0, + infinities: 0 }; if (result.rows.length) { formattedResult.operation = this.operation; formattedResult.result = result.rows[0].result; formattedResult.nulls = result.rows[0].nulls_count; + formattedResult.nans = result.rows[0].nans_count; + formattedResult.infinities = result.rows[0].infinities_count; } return formattedResult; diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index d23519ae..18fc1240 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -248,7 +248,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 5, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -260,7 +267,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -407,7 +421,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -419,7 +440,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); From 0aae29fb4b64e8dddb7baa3c136fda8db9ea830a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 9 Jun 2017 15:28:18 +0200 Subject: [PATCH 05/60] Fix jshint typo --- lib/cartodb/models/dataview/formula.js | 6 ++++-- test/acceptance/dataviews/formula.js | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 1421032d..04b89835 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -10,8 +10,10 @@ var formulaQueryTpl = dot.template([ ' {{=it._operation}}({{=it._column}}) AS result,', ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', ' {{?it._operation!==\'count\'}}', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', ' {{?}}', 'FROM ({{=it._query}}) _cdb_formula', '{{?it._operation!==\'count\'}}', diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js index 93db6b38..585a3b82 100644 --- a/test/acceptance/dataviews/formula.js +++ b/test/acceptance/dataviews/formula.js @@ -71,7 +71,7 @@ describe('formula-dataview: special float valuer', function() { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('val_formula', {}, function(err, dataview) { assert.ok(!err, err); - assert.equal(dataview.result, 501) + assert.equal(dataview.result, 501); assert.ok(dataview.infinities === (250 + 250)); assert.ok(dataview.nans === 250); done(); From e6aededf089fff0bb326ad291453d4585f400163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 17:19:05 +0200 Subject: [PATCH 06/60] Fix typo --- test/acceptance/dataviews/formula.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js index 585a3b82..941dc07e 100644 --- a/test/acceptance/dataviews/formula.js +++ b/test/acceptance/dataviews/formula.js @@ -11,7 +11,7 @@ function createMapConfig(layers, dataviews, analysis) { }; } -describe('formula-dataview: special float valuer', function() { +describe('formula-dataview: special float values', function() { afterEach(function(done) { if (this.testClient) { From ba6dc62a38245f1e18c47600ba165fb312d55e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 18:15:39 +0200 Subject: [PATCH 07/60] Going red: implementet test to check special float values support --- test/acceptance/dataviews/aggregation.js | 82 ++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 259cd2af..988f71ff 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -146,3 +146,85 @@ describe('aggregations happy cases', function() { }); }); }); + +describe.only('aggregation-dataview: special float values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_aggregation: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_aggregation', {}, function(err, dataview) { + assert.ifError(err); + assert.equal(dataview.result, 501); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); From e60bb770db7d0e6fe836a927d16db23df58a04d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 18:55:33 +0200 Subject: [PATCH 08/60] Summarizes infinity and NaN values --- lib/cartodb/models/dataview/aggregation.js | 83 +++++++++++++++------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index c15f0506..8c6fff54 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -5,11 +5,28 @@ var debug = require('debug')('windshaft:widget:aggregation'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' AND', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' count(1) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float THEN 1 ELSE 0 END) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -57,9 +74,9 @@ var rankedAggregationQueryTpl = dot.template([ var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'ORDER BY value DESC' ].join('\n')); @@ -128,6 +145,8 @@ Aggregation.prototype.sql = function(psql, override, callback) { var aggregationSql; + + if (!!override.ownFilter) { aggregationSql = [ this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), @@ -157,28 +176,34 @@ Aggregation.prototype.sql = function(psql, override, callback) { Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { return [ - "WITH", - [ - summaryQueryTpl({ - _query: query, - _column: column - }), - rankedCategoriesQueryTpl({ - _query: query, - _column: column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: query, - _column: column - }), - categoriesSummaryCountQueryTpl({ - _query: query, - _column: column - }) - ].join(',\n') - ].join('\n'); + "WITH", + [ + filteredQueryTpl({ + _query: this.query, + _column: this.column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + summaryQueryTpl({ + _query: query, + _column: column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + rankedCategoriesQueryTpl({ + _query: query, + _column: column, + _aggregation: this.getAggregationSql(), + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + categoriesSummaryMinMaxQueryTpl({ + _query: query, + _column: column + }), + categoriesSummaryCountQueryTpl({ + _query: query, + _column: column + }) + ].join(',\n') + ].join('\n'); }; var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); @@ -193,6 +218,8 @@ Aggregation.prototype.format = function(result) { var categories = []; var count = 0; var nulls = 0; + var nans = 0; + var infinities = 0; var minValue = 0; var maxValue = 0; var categoriesCount = 0; @@ -202,12 +229,14 @@ Aggregation.prototype.format = function(result) { var firstRow = result.rows[0]; count = firstRow.count; nulls = firstRow.nulls_count; + nans = firstRow.nans_count; + infinities = firstRow.infinities_count; minValue = firstRow.min_val; maxValue = firstRow.max_val; categoriesCount = firstRow.categories_count; result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count', 'nans_count', 'infinities_count')); }); } @@ -215,6 +244,8 @@ Aggregation.prototype.format = function(result) { aggregation: this.aggregation, count: count, nulls: nulls, + nans: nans, + infinities: infinities, min: minValue, max: maxValue, categoriesCount: categoriesCount, From 7b5111614c89e1b61f8dbb14cd701edb9ec98c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:21:41 +0200 Subject: [PATCH 09/60] Summarize special float values for ranked aggregation --- lib/cartodb/models/dataview/aggregation.js | 8 ++++---- test/acceptance/dataviews/aggregation.js | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 8c6fff54..145b6e9d 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -61,15 +61,15 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', - ' count, categories_count', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' ].join('\n')); var aggregationQueryTpl = dot.template([ @@ -145,7 +145,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { var aggregationSql; - + console.log('override.ownFilter', override.ownFilter); if (!!override.ownFilter) { aggregationSql = [ diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 988f71ff..d4f3d65c 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -217,14 +217,16 @@ describe.only('aggregation-dataview: special float values', function() { ] ); - it('should filter infinities out and count them in the summary', function(done) { - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('val_aggregation', {}, function(err, dataview) { - assert.ifError(err); - assert.equal(dataview.result, 501); - assert.ok(dataview.infinities === (250 + 250)); - assert.ok(dataview.nans === 250); - done(); + [{ own_filter: 0 }, {}].forEach(function (filter) { + it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.equal(dataview.result, 501); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); }); - }); + }) }); From 75d07745e6496d4bef923381196f22c0d1758572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:22:37 +0200 Subject: [PATCH 10/60] Improve readability --- test/acceptance/dataviews/aggregation.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index d4f3d65c..02b19782 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -217,7 +217,8 @@ describe.only('aggregation-dataview: special float values', function() { ] ); - [{ own_filter: 0 }, {}].forEach(function (filter) { + var filters = [{ own_filter: 0 }, {}]; + filters.forEach(function (filter) { it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { From 962fa055747f25548b23479b4c746cc2c25ae63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:28:49 +0200 Subject: [PATCH 11/60] Remove console --- lib/cartodb/models/dataview/aggregation.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 145b6e9d..f06f6091 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -145,8 +145,6 @@ Aggregation.prototype.sql = function(psql, override, callback) { var aggregationSql; - console.log('override.ownFilter', override.ownFilter); - if (!!override.ownFilter) { aggregationSql = [ this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), From 8b2fa27ba75f5f488cc7f6ffc7501f66a747d761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:45:06 +0200 Subject: [PATCH 12/60] Calculate aggregation filtering out special float values --- lib/cartodb/models/dataview/aggregation.js | 2 +- test/acceptance/dataviews/aggregation.js | 23 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index f06f6091..9ce889da 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -35,7 +35,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 02b19782..71aec370 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -187,7 +187,7 @@ describe.only('aggregation-dataview: special float values', function() { type: 'aggregation', options: { column: 'cat', - aggregation: 'sum', + aggregation: 'avg', aggregationColumn: 'val' } } @@ -217,15 +217,34 @@ describe.only('aggregation-dataview: special float values', function() { ] ); + // the_geom_webmercator | val | cat + // ----------------------+-----------+------------ + // | -Infinity | category_2 + // | NaN | category_1 + // | 3 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 7 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 11 | category_2 + // | " | " + var filters = [{ own_filter: 0 }, {}]; filters.forEach(function (filter) { it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { assert.ifError(err); - assert.equal(dataview.result, 501); assert.ok(dataview.infinities === (250 + 250)); assert.ok(dataview.nans === 250); + assert.ok(dataview.categories.length === 1); + dataview.categories.forEach(function (category) { + assert.ok(category.category === 'category_2'); + assert.ok(category.value === 501); + }) done(); }); }); From cb7ec5d556e4020b697fd82b71e270cc095a8988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:49:58 +0200 Subject: [PATCH 13/60] Fix jshint typos --- lib/cartodb/models/dataview/aggregation.js | 18 +++++++++++++----- test/acceptance/dataviews/aggregation.js | 4 ++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 9ce889da..6e46ea31 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -25,7 +25,13 @@ var summaryQueryTpl = dot.template([ ' SELECT', ' count(1) AS count,', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float THEN 1 ELSE 0 END) AS infinities_count,', + ' sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' @@ -61,12 +67,13 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', + ' min_val, max_val, count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' @@ -234,7 +241,8 @@ Aggregation.prototype.format = function(result) { categoriesCount = firstRow.categories_count; result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count', 'nans_count', 'infinities_count')); + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', + 'max_val', 'categories_count', 'nans_count', 'infinities_count')); }); } diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 71aec370..4f7c0062 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -244,9 +244,9 @@ describe.only('aggregation-dataview: special float values', function() { dataview.categories.forEach(function (category) { assert.ok(category.category === 'category_2'); assert.ok(category.value === 501); - }) + }); done(); }); }); - }) + }); }); From 227937bf4c933488728587ad8e034f4ce248a20c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 12 Jun 2017 19:50:24 +0200 Subject: [PATCH 14/60] Remove test filter --- test/acceptance/dataviews/aggregation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 4f7c0062..5d9dc499 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -147,7 +147,7 @@ describe('aggregations happy cases', function() { }); }); -describe.only('aggregation-dataview: special float values', function() { +describe('aggregation-dataview: special float values', function() { afterEach(function(done) { if (this.testClient) { From 3ae66e41437160dfd3cf9bb2cac52cb0df195d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 09:30:43 +0200 Subject: [PATCH 15/60] Do not filter special values out if aggregation column is not defined --- lib/cartodb/models/dataview/aggregation.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 6e46ea31..71574ac5 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -11,12 +11,14 @@ var filteredQueryTpl = dot.template([ ' FROM ({{=it._query}}) _cdb_filtered_source', ' WHERE', ' {{=it._column}} IS NOT NULL', + ' {{?it._aggregationColumn}}', ' AND', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'NaN\'::float', + ' {{?}}', ')' ].join(' \n')); From 551b6d409a1820fd0a2acc534ff02c246c51e777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 09:44:53 +0200 Subject: [PATCH 16/60] Remove bad condition --- lib/cartodb/models/dataview/aggregation.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 71574ac5..e61b1454 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -9,10 +9,8 @@ var filteredQueryTpl = dot.template([ 'filtered_source AS (', ' SELECT *', ' FROM ({{=it._query}}) _cdb_filtered_source', - ' WHERE', - ' {{=it._column}} IS NOT NULL', ' {{?it._aggregationColumn}}', - ' AND', + ' WHERE', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', From c08db78a0bc496ea290180eed210091af1f4eba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 19:01:28 +0200 Subject: [PATCH 17/60] Going red: implement test to check aggregation with overviews support special float values --- test/acceptance/dataviews/overviews.js | 58 +++++++++++++++++++++ test/support/sql/windshaft.test.sql | 72 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 18fc1240..c822caee 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() { params: { query: 'select * from test_table_overviews' } + }, + { + id: 'data-source-special-float-values', + type: 'source', + params: { + query: 'select * from test_special_float_values_table_overviews' + } } ], dataviews: { @@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() { aggregationColumn: 'name', } }, + test_categories_special_values: { + type: 'aggregation', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'name', + aggregation: 'sum', + aggregationColumn: 'value', + } + }, test_histogram: { type: 'histogram', source: {id: 'data-source'}, @@ -202,6 +220,17 @@ describe('dataviews using tables with overviews', function() { cartocss_version: '2.3.0', source: { id: 'data-source' } } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_special_float_values_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { + id: 'data-source-special-float-values' + } + } } ] }; @@ -473,5 +502,34 @@ describe('dataviews using tables with overviews', function() { }); + describe('aggregation special float values', function () { + var params = {}; + + it("should expose an aggregation dataview", function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_categories_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + aggregation: 'sum', + count: 5, + nulls: 0, + nans: 1, + infinities: 1, + min: 6, + max: 6, + categoriesCount: 3, + categories:[ + { category: 'El Rey del Tallarín', value: null, agg: false }, + { category: 'El Lacón', value: null, agg: false }, + { category: 'Hawai', value: 6, agg: false } + ], + type: 'aggregation' + }); + testClient.drain(done); + }); + }); + }); }); }); diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 91484950..996c6b3e 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES INSERT INTO _vovw_1_test_table_overviews VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5); +-- table with overviews whit special float values + +CREATE TABLE test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id; + +SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true); + +ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass); + +INSERT INTO test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1), +(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1), +(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1); + +ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id); + +CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom); +CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE TABLE _vovw_1_test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER; + +INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2); -- analysis tables ----------------------------------------------- From 668b22628ce4ce5cfbf589ef50ab4868e549c11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 19:01:59 +0200 Subject: [PATCH 18/60] Going green: support special float values in aggergation dataview with overviews --- .../models/dataview/overviews/aggregation.js | 63 +++++++++++++++---- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index da63b27f..1b0e2ec6 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,14 +1,38 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../aggregation'); +var debug = require('debug')('windshaft:widget:aggregation:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' {{?it._aggregationColumn}}', + ' WHERE', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float', + ' {{?}}', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' sum(_feature_count) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', + ' sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -17,7 +41,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', @@ -43,21 +67,23 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'ORDER BY value DESC' ].join('\n')); @@ -84,21 +110,28 @@ Aggregation.prototype.sql = function(psql, override, callback) { } var _query = this.rewrittenQuery(this.query); + var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ "WITH", [ + filteredQueryTpl({ + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn + }), summaryQueryTpl({ _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -120,15 +153,21 @@ Aggregation.prototype.sql = function(psql, override, callback) { aggregationSql = [ "WITH", [ + filteredQueryTpl({ + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn + }), summaryQueryTpl({ _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -147,6 +186,8 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } + debug(aggregationSql); + return callback(null, aggregationSql); }; From 3f17c8b15ab9074b1004d08aee23f59656fba42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 15:05:46 +0200 Subject: [PATCH 19/60] Filter out special float values before categorizing them --- lib/cartodb/models/dataview/overviews/aggregation.js | 4 ++-- test/acceptance/dataviews/overviews.js | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 1b0e2ec6..cc95e73c 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -60,7 +60,7 @@ var categoriesSummaryCountQueryTpl = dot.template([ ' SELECT count(1) AS categories_count', ' FROM (', ' SELECT {{=it._column}} AS category', - ' FROM ({{=it._query}}) _cdb_categories', + ' FROM filtered_source', ' GROUP BY {{=it._column}}', ' ) _cdb_categories_count', ')' @@ -82,7 +82,7 @@ var rankedAggregationQueryTpl = dot.template([ var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', - 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', + 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count', 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'ORDER BY value DESC' ].join('\n')); diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index c822caee..08ee6b7c 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -505,7 +505,7 @@ describe('dataviews using tables with overviews', function() { describe('aggregation special float values', function () { var params = {}; - it("should expose an aggregation dataview", function (done) { + it("should expose an aggregation dataview filtering special float values out", function (done) { var testClient = new TestClient(overviewsMapConfig); testClient.getDataview('test_categories_special_values', params, function (err, dataview) { if (err) { @@ -519,12 +519,8 @@ describe('dataviews using tables with overviews', function() { infinities: 1, min: 6, max: 6, - categoriesCount: 3, - categories:[ - { category: 'El Rey del Tallarín', value: null, agg: false }, - { category: 'El Lacón', value: null, agg: false }, - { category: 'Hawai', value: 6, agg: false } - ], + categoriesCount: 1, + categories: [ { category: 'Hawai', value: 6, agg: false } ], type: 'aggregation' }); testClient.drain(done); From 06d40e8b1e0288e3a6d477c90f40c7fa03869274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 18:57:26 +0200 Subject: [PATCH 20/60] Going red: add test to check that special float values are not being filtered out when the layer uses overviews --- test/acceptance/dataviews/overviews.js | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 08ee6b7c..20a7ca9d 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -178,6 +178,16 @@ describe('dataviews using tables with overviews', function() { bins: 2 } }, + test_histogram_special_values: { + type: 'histogram', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + bins: 2 + } + }, test_avg: { type: 'formula', source: {id: 'data-source'}, @@ -526,6 +536,27 @@ describe('dataviews using tables with overviews', function() { testClient.drain(done); }); }); + + it('should expose a histogram dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_histogram_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + bin_width: 0, + bins_count: 1, + bins_start: 3, + nulls: 0, + infinities: 1, + nans: 1, + avg: 3, + bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ], + type: 'histogram' + }); + testClient.drain(done); + }); + }); }); }); }); From dee00e6abd10e78583c3d0871e26f73bda989e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:00:37 +0200 Subject: [PATCH 21/60] Going green: handle special float values for histogram when overviews are involved --- .../models/dataview/overviews/histogram.js | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 67da4514..ed88a621 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -12,12 +12,27 @@ var columnTypeQueryTpl = dot.template( var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join(' \n')); @@ -26,7 +41,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join('\n')); @@ -37,7 +52,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -56,7 +71,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -76,11 +91,34 @@ var nullsQueryTpl = dot.template([ ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + 'infinities AS (', + ' SELECT', + ' count(*) AS infinities_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + 'nans AS (', + ' SELECT', + ' count(*) AS nans_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', + ' infinities_count,', + ' nans_count,', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -90,9 +128,8 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', ' sum(_feature_count) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'FROM filtered_source, basics, nulls, infinities, nans, bins', + 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', 'ORDER BY bin' ].join('\n')); @@ -156,7 +193,12 @@ Histogram.prototype.sql = function(psql, override, callback) { var _query = this.rewrittenQuery(this.query); - var basicsQuery, binsQuery; + var filteredQuery, basicsQuery, binsQuery; + + filteredQuery = filteredQueryTpl({ + _query: _query, + _column: _column + }); if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { basicsQuery = overrideBasicsQueryTpl({ @@ -202,11 +244,20 @@ Histogram.prototype.sql = function(psql, override, callback) { var histogramSql = [ "WITH", [ + filteredQuery, basicsQuery, binsQuery, nullsQueryTpl({ _query: _query, _column: _column + }), + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ + _query: _query, + _column: _column }) ].join(',\n'), histogramQueryTpl({ From ef849aec342a588ee359f9af70e83480a2a90d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:19:08 +0200 Subject: [PATCH 22/60] Going red: add test to check that special float values are not being filtered out in formula dataview when the layer uses overviews --- test/acceptance/dataviews/overviews.js | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 20a7ca9d..a024febd 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -196,6 +196,16 @@ describe('dataviews using tables with overviews', function() { operation: 'avg' } }, + test_sum_special_values: { + type: 'formula', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + operation: 'sum' + } + }, test_count: { type: 'formula', source: {id: 'data-source'}, @@ -557,6 +567,24 @@ describe('dataviews using tables with overviews', function() { testClient.drain(done); }); }); + + it('should expose a formula (sum) dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_sum_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + operation: 'sum', + result: 6, + nulls: 0, + nans: 1, + infinities: 1, + type: 'formula' + }); + testClient.drain(done); + }); + }); }); }); }); From 81f60959e59901e8f412d9f13a83f17aee62a230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:20:39 +0200 Subject: [PATCH 23/60] Going green: handle special float values for formula when overviews are involved --- .../models/dataview/overviews/formula.js | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 9e331f0b..ba98b2c8 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -14,14 +14,34 @@ var formulaQueryTpls = { 'sum': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), 'avg': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), }; From 7d0af4e2596982fb154d2ad4bf495f11d9d254fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 14 Jun 2017 19:20:39 +0200 Subject: [PATCH 24/60] Going green: handle special float values for formula when overviews are involved --- .../models/dataview/overviews/formula.js | 28 ++++++-- test/acceptance/dataviews/overviews.js | 65 ++++++++++++++++--- 2 files changed, 80 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 9e331f0b..ba98b2c8 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -14,14 +14,34 @@ var formulaQueryTpls = { 'sum': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), 'avg': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + 'FROM ({{=it._query}}) _cdb_formula', + 'WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')), }; diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index a024febd..4362d7a3 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -196,7 +196,7 @@ describe('dataviews using tables with overviews', function() { operation: 'avg' } }, - test_sum_special_values: { + test_formula_sum_special_values: { type: 'formula', source: { id: 'data-source-special-float-values' @@ -261,7 +261,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -273,7 +280,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":3, + "nulls":0, + "type":"formula", + "infinities": 0, + "nans": 0 + }); testClient.drain(done); }); @@ -285,7 +299,12 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":5, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -338,7 +357,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -435,7 +461,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -446,7 +479,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -514,7 +554,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -570,7 +617,7 @@ describe('dataviews using tables with overviews', function() { it('should expose a formula (sum) dataview filtering special float values out', function (done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_sum_special_values', params, function (err, dataview) { + testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) { if (err) { return done(err); } From 443c1100d7487b2f2d29a8a023eb3bf0d7675f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 16:31:24 +0200 Subject: [PATCH 25/60] Formula dataview: support special values only if column is a float column --- lib/cartodb/models/dataview/formula.js | 45 +++++++++--- .../models/dataview/overviews/formula.js | 68 +++++++++++++++---- test/acceptance/dataviews/overviews.js | 13 +++- 3 files changed, 102 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 04b89835..1d3204e7 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -5,25 +5,25 @@ var debug = require('debug')('windshaft:widget:formula'); var dot = require('dot'); dot.templateSettings.strip = false; +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); + var formulaQueryTpl = dot.template([ 'SELECT', ' {{=it._operation}}({{=it._column}}) AS result,', ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - ' {{?it._operation!==\'count\'}}', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', - ' {{?}}', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._operation!==\'count\'}}', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float', - '{{?}}' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')); var VALID_OPERATIONS = { @@ -34,6 +34,11 @@ var VALID_OPERATIONS = { max: true }; +var FLOAT_OIDS = { + 700: true, + 701: true +}; + var TYPE = 'formula'; /** @@ -63,6 +68,7 @@ function Formula(query, options) { this.query = query; this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = new BaseWidget(); @@ -71,13 +77,36 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } var _query = this.query; + + var columnTypeQuery = columnTypeQueryTpl({ + column: this.column, query: _query + }); + + if (this._isFloatColumn === null) { + var readOnlyTransaction = true; + psql.query(columnTypeQuery, function(err, result) { + self._isFloatColumn = false; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (FLOAT_OIDS.hasOwnProperty(pgType)) { + self._isFloatColumn = true; + } + } + self.sql(psql, override, callback); + }, readOnlyTransaction); + return null; + } + var formulaSql = formulaQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _operation: this.operation, _column: this.column diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index ba98b2c8..a1437c73 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,47 +1,61 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); +var debug = require('debug')('windshaft:widget:formula:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var FLOAT_OIDS = { + 700: true, + 701: true +}; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); + var formulaQueryTpls = { 'count': dot.template([ 'SELECT', 'sum(_feature_count) AS result,', '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula' ].join('\n')), 'sum': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')), 'avg': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')), }; @@ -49,6 +63,7 @@ function Formula(query, options, queryRewriter, queryRewriteData, params) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -57,20 +72,45 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; var formulaQueryTpl = formulaQueryTpls[this.operation]; if ( formulaQueryTpl ) { // supported formula for use with overviews + + var columnTypeQuery = columnTypeQueryTpl({ + column: this.column, query: this.query + }); + + if (this._isFloatColumn === null) { + var readOnlyTransaction = true; + psql.query(columnTypeQuery, function(err, result) { + self._isFloatColumn = false; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (FLOAT_OIDS.hasOwnProperty(pgType)) { + self._isFloatColumn = true; + } + } + self.sql(psql, override, callback); + }, readOnlyTransaction); + return null; + } + var formulaSql = formulaQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: this.rewrittenQuery(this.query), _operation: this.operation, _column: this.column }); callback = callback || override; + debug(formulaSql); + return callback(null, formulaSql); } + // default behaviour return this.defaultSql(psql, override, callback); }; diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 4362d7a3..dcff687f 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -303,7 +303,9 @@ describe('dataviews using tables with overviews', function() { "operation":"count", "result":5, "nulls":0, - "type":"formula" + "type":"formula", + "infinities": 0, + "nans": 0 }); testClient.drain(done); @@ -498,7 +500,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":1, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); From ad570ab6f23d8b5b5e017dcecf75b09e4af7ee0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 18:04:35 +0200 Subject: [PATCH 26/60] Use dataview base to get column type in formula dataviews --- lib/cartodb/models/dataview/base.js | 34 +++++ lib/cartodb/models/dataview/formula.js | 31 +---- .../models/dataview/overviews/formula.js | 121 ++++++++---------- 3 files changed, 92 insertions(+), 94 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index b2e2f188..d7479b6d 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -1,3 +1,6 @@ +var dot = require('dot'); +dot.templateSettings.strip = false; + function BaseDataview() {} module.exports = BaseDataview; @@ -24,3 +27,34 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { BaseDataview.prototype.search = function(psql, userQuery, callback) { return callback(null, this.format({ rows: [] })); }; + +var FLOAT_OIDS = { + 700: true, + 701: true +}; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1' +); + +BaseDataview.prototype.getColumnType = function (psql, column, query, callback) { + var readOnlyTransaction = true; + + var columnTypeQuery = columnTypeQueryTpl({ + column: column, query: query + }); + + psql.query(columnTypeQuery, function(err, result) { + if (err) { + return callback(err); + } + var pgType = result.rows[0].pg_typeof; + callback(null, getPGTypeName(pgType)); + }, readOnlyTransaction); +}; + +function getPGTypeName (pgType) { + return { + float: FLOAT_OIDS.hasOwnProperty(pgType) + }; +} diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 1d3204e7..2272089c 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -5,10 +5,6 @@ var debug = require('debug')('windshaft:widget:formula'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var formulaQueryTpl = dot.template([ 'SELECT', ' {{=it._operation}}({{=it._column}}) AS result,', @@ -34,11 +30,6 @@ var VALID_OPERATIONS = { max: true }; -var FLOAT_OIDS = { - 700: true, - 701: true -}; - var TYPE = 'formula'; /** @@ -84,30 +75,20 @@ Formula.prototype.sql = function(psql, override, callback) { override = {}; } - var _query = this.query; - - var columnTypeQuery = columnTypeQueryTpl({ - column: this.column, query: _query - }); - if (this._isFloatColumn === null) { - var readOnlyTransaction = true; - psql.query(columnTypeQuery, function(err, result) { - self._isFloatColumn = false; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (FLOAT_OIDS.hasOwnProperty(pgType)) { - self._isFloatColumn = true; - } + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.query, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; } self.sql(psql, override, callback); - }, readOnlyTransaction); + }); return null; } var formulaSql = formulaQueryTpl({ _isFloatColumn: this._isFloatColumn, - _query: _query, + _query: this.query, _operation: this.operation, _column: this.column }); diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index a1437c73..533dd921 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -5,58 +5,49 @@ var debug = require('debug')('windshaft:widget:formula:overview'); var dot = require('dot'); dot.templateSettings.strip = false; -var FLOAT_OIDS = { - 700: true, - 701: true -}; - -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var formulaQueryTpls = { - 'count': dot.template([ - 'SELECT', - 'sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), - 'sum': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' - ].join('\n')), - 'avg': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' - ].join('\n')), + 'count': dot.template([ + 'SELECT', + 'sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'sum': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), + 'avg': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), }; function Formula(query, options, queryRewriter, queryRewriteData, params) { @@ -71,38 +62,30 @@ Formula.prototype.constructor = Formula; module.exports = Formula; -Formula.prototype.sql = function(psql, override, callback) { +Formula.prototype.sql = function (psql, override, callback) { var self = this; var formulaQueryTpl = formulaQueryTpls[this.operation]; - if ( formulaQueryTpl ) { + if (formulaQueryTpl) { // supported formula for use with overviews - - var columnTypeQuery = columnTypeQueryTpl({ - column: this.column, query: this.query - }); - if (this._isFloatColumn === null) { - var readOnlyTransaction = true; - psql.query(columnTypeQuery, function(err, result) { - self._isFloatColumn = false; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (FLOAT_OIDS.hasOwnProperty(pgType)) { - self._isFloatColumn = true; - } + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.query, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; } self.sql(psql, override, callback); - }, readOnlyTransaction); + }); return null; } var formulaSql = formulaQueryTpl({ _isFloatColumn: this._isFloatColumn, - _query: this.rewrittenQuery(this.query), + _query: this.rewrittenQuery(this.query), _operation: this.operation, - _column: this.column + _column: this.column }); + callback = callback || override; debug(formulaSql); From 115d8fe6854f6971690a2a8ad7d55b02ed802b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 19:07:31 +0200 Subject: [PATCH 27/60] Handle special float values only when aggregation columns is float --- lib/cartodb/models/dataview/aggregation.js | 63 ++++++++++++++++------ 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index e61b1454..0d576614 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -9,14 +9,12 @@ var filteredQueryTpl = dot.template([ 'filtered_source AS (', ' SELECT *', ' FROM ({{=it._query}}) _cdb_filtered_source', - ' {{?it._aggregationColumn}}', - ' WHERE', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', ' AND', - ' {{=it._aggregationColumn}} != \'NaN\'::float', - ' {{?}}', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -24,15 +22,15 @@ var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' count(1) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', - ' sum(', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', ' CASE', ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', ' THEN 1', ' ELSE 0', ' END', ' ) AS infinities_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -68,22 +66,24 @@ var categoriesSummaryCountQueryTpl = dot.template([ var rankedAggregationQueryTpl = dot.template([ 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', - ' min_val, max_val, count, categories_count, nans_count, infinities_count', + ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); @@ -108,7 +108,7 @@ var TYPE = 'aggregation'; } } */ -function Aggregation(query, options) { +function Aggregation(query, options, queries) { if (!_.isString(options.column)) { throw new Error('Aggregation expects `column` in widget options'); } @@ -132,9 +132,11 @@ function Aggregation(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = new BaseWidget(); @@ -143,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var _query = this.query; var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -164,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } else { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', @@ -179,16 +208,18 @@ Aggregation.prototype.sql = function(psql, override, callback) { return callback(null, aggregationSql); }; -Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { +Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { return [ "WITH", [ filteredQueryTpl({ + _isFloatColumn: isFloatColumn, _query: this.query, _column: this.column, _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null }), summaryQueryTpl({ + _isFloatColumn: isFloatColumn, _query: query, _column: column, _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null From 20d7f1a7c56749d8e18acb930c58b558c7b1bc12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 19:22:26 +0200 Subject: [PATCH 28/60] Handle special float values only when aggregation columns is float (overviews) --- .../models/dataview/overviews/aggregation.js | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index cc95e73c..730ba822 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -9,14 +9,12 @@ var filteredQueryTpl = dot.template([ 'filtered_source AS (', ' SELECT *', ' FROM ({{=it._query}}) _cdb_filtered_source', - ' {{?it._aggregationColumn}}', - ' WHERE', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', ' {{=it._aggregationColumn}} != \'infinity\'::float', ' AND', ' {{=it._aggregationColumn}} != \'-infinity\'::float', ' AND', - ' {{=it._aggregationColumn}} != \'NaN\'::float', - ' {{?}}', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -24,15 +22,15 @@ var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' sum(_feature_count) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', - ' sum(', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', ' CASE', ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', ' THEN 1', ' ELSE 0', ' END', ' ) AS infinities_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -68,34 +66,38 @@ var categoriesSummaryCountQueryTpl = dot.template([ var rankedAggregationQueryTpl = dot.template([ 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', - ' count, categories_count, nans_count, infinities_count', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); var CATEGORIES_LIMIT = 6; - function Aggregation(query, options, queryRewriter, queryRewriteData, params) { + function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -104,6 +106,8 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; @@ -112,17 +116,30 @@ Aggregation.prototype.sql = function(psql, override, callback) { var _query = this.rewrittenQuery(this.query); var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ "WITH", [ filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn }), summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn @@ -143,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -154,11 +172,13 @@ Aggregation.prototype.sql = function(psql, override, callback) { "WITH", [ filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn }), summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationColumn: _aggregationColumn @@ -179,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _limit: CATEGORIES_LIMIT From b1ac5b8ca9475dddb859a2e539e0538d062c0f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 16 Jun 2017 12:57:46 +0200 Subject: [PATCH 29/60] Handle special float values only if column is float --- lib/cartodb/models/dataview/base.js | 9 +- lib/cartodb/models/dataview/histogram.js | 101 +++++++++-------- .../models/dataview/overviews/histogram.js | 104 ++++++++++-------- 3 files changed, 120 insertions(+), 94 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index d7479b6d..2cd24261 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -33,6 +33,12 @@ var FLOAT_OIDS = { 701: true }; +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + var columnTypeQueryTpl = dot.template( 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1' ); @@ -55,6 +61,7 @@ BaseDataview.prototype.getColumnType = function (psql, column, query, callback) function getPGTypeName (pgType) { return { - float: FLOAT_OIDS.hasOwnProperty(pgType) + float: FLOAT_OIDS.hasOwnProperty(pgType), + date: DATE_OIDS.hasOwnProperty(pgType) }; } diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index f6b2635d..73cc8724 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -5,9 +5,6 @@ var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; @@ -19,12 +16,12 @@ var filteredQueryTpl = dot.template([ ' FROM ({{=it._query}}) _cdb_filtered_source', ' WHERE', ' {{=it._column}} IS NOT NULL', - ' AND', + ' {{?it._isFloatColumn}}AND', ' {{=it._column}} != \'infinity\'::float', ' AND', ' {{=it._column}} != \'-infinity\'::float', ' AND', - ' {{=it._column}} != \'NaN\'::float', + ' {{=it._column}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -118,8 +115,8 @@ var histogramQueryTpl = dot.template([ ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', - ' infinities_count,', - ' nans_count,', + ' {{?it._isFloatColumn}}infinities_count,', + ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -129,8 +126,9 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, basics, nulls, infinities, nans, bins', - 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', + 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'GROUP BY bin, bins_number, bin_width, nulls_count,', + ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -164,55 +162,49 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.queries.no_filters - }); - if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction return null; } + var histogramSql = this._buildQuery(override); + + return callback(null, histogramSql); +}; + +Histogram.prototype._buildQuery = function (override) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; + var _query = this.query; + if (this._columnType === 'date') { _column = columnCastTpl({column: _column}); } - var _query = this.query; - - var filteredQuery, basicsQuery, binsQuery; - filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }); - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, @@ -232,7 +224,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -253,17 +245,18 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }), + if (this._columnType === 'float') { + cteSql.push( infinitiesQueryTpl({ _query: _query, _column: _column @@ -272,8 +265,14 @@ Histogram.prototype.sql = function(psql, override, callback) { _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) @@ -281,7 +280,15 @@ Histogram.prototype.sql = function(psql, override, callback) { debug(histogramSql); - return callback(null, histogramSql); + return histogramSql; +}; + +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); }; Histogram.prototype.format = function(result, override) { diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index ed88a621..361cc8f2 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -1,14 +1,11 @@ var _ = require('underscore'); var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../histogram'); +var debug = require('debug')('windshaft:dataview:histogram:overview'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -18,12 +15,12 @@ var filteredQueryTpl = dot.template([ ' FROM ({{=it._query}}) _cdb_filtered_source', ' WHERE', ' {{=it._column}} IS NOT NULL', - ' AND', + ' {{?it._isFloatColumn}}AND', ' {{=it._column}} != \'infinity\'::float', ' AND', ' {{=it._column}} != \'-infinity\'::float', ' AND', - ' {{=it._column}} != \'NaN\'::float', + ' {{=it._column}} != \'NaN\'::float{{?}}', ')' ].join(' \n')); @@ -117,8 +114,8 @@ var histogramQueryTpl = dot.template([ ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', - ' infinities_count,', - ' nans_count,', + ' {{?it._isFloatColumn}}infinities_count,', + ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -128,8 +125,9 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', ' sum(_feature_count) AS freq', - 'FROM filtered_source, basics, nulls, infinities, nans, bins', - 'GROUP BY bin, bins_number, bin_width, nulls_count, infinities_count, nans_count, avg_val', + 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + ' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -149,36 +147,23 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; - -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.rewrittenQuery(this.queries.no_filters) - }); if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction @@ -191,16 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) { return this.defaultSql(psql, override, callback); } + var histogramSql = this._buildQuery(override); + + return callback(null, histogramSql); +}; + +Histogram.prototype._buildQuery = function (override) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; var _query = this.rewrittenQuery(this.query); - var filteredQuery, basicsQuery, binsQuery; - filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }); - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { + debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, _column: _column, @@ -219,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -240,17 +233,18 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }), + if (this._columnType === 'float') { + cteSql.push( infinitiesQueryTpl({ _query: _query, _column: _column @@ -259,12 +253,30 @@ Histogram.prototype.sql = function(psql, override, callback) { _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) ].join('\n'); - return callback(null, histogramSql); + debug(histogramSql); + + return histogramSql; }; + +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); +}; + + From e44d418db39dec200ffcdfa8ca6d03f0d717a40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Jun 2017 18:44:21 +0200 Subject: [PATCH 30/60] Going red: Add test to check that aggregation dataview supports numeric special value --- test/acceptance/dataviews/aggregation.js | 46 ++++++++++++++++++++++++ test/support/test-client.js | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 5d9dc499..86eb0613 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -190,6 +190,17 @@ describe('aggregation-dataview: special float values', function() { aggregation: 'avg', aggregationColumn: 'val' } + }, + sum_aggregation_numeric: { + source: { + id: 'a1' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } } }, [ @@ -213,10 +224,31 @@ describe('aggregation-dataview: special float values', function() { 'FROM generate_series(1, 1000) x' ].join('\n') } + }, { + "id": "a1", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 3 = 0 THEN \'NaN\'::numeric', + ' WHEN x % 3 = 1 THEN x', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } } ] ); + // Source a0 + // ----------------------------------------------- // the_geom_webmercator | val | cat // ----------------------+-----------+------------ // | -Infinity | category_2 @@ -248,5 +280,19 @@ describe('aggregation-dataview: special float values', function() { done(); }); }); + + it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.ok(dataview.nans === 333); + assert.ok(dataview.categories.length === 2); + dataview.categories.forEach(function (category) { + assert.ok(category.value !== null); + }); + done(); + }); + }); + }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 04980a5a..659d6686 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } params.placeholders = params.placeholders || {}; - + assert.response(server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), From 5555b8ad8ec76025cb1770336271e1802b526477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Jun 2017 18:59:36 +0200 Subject: [PATCH 31/60] Going green: support numeric NaN values for dataviews --- lib/cartodb/models/dataview/base.js | 3 ++- test/acceptance/dataviews/aggregation.js | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 2cd24261..f8eb1027 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -30,7 +30,8 @@ BaseDataview.prototype.search = function(psql, userQuery, callback) { var FLOAT_OIDS = { 700: true, - 701: true + 701: true, + 1700: true }; var DATE_OIDS = { diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 86eb0613..697004e8 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -293,6 +293,5 @@ describe('aggregation-dataview: special float values', function() { done(); }); }); - }); }); From 14e71b929abf24674cfe654de12d6377c8173b15 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 22 Jun 2017 17:48:55 +0200 Subject: [PATCH 32/60] Release version 3.9.4 --- NEWS.md | 6 ++ package.json | 4 +- yarn.lock | 153 ++++++++++++++++++++++++++------------------------- 3 files changed, 85 insertions(+), 78 deletions(-) diff --git a/NEWS.md b/NEWS.md index 87ec5032..9f334355 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.9.4 +Released 2017-06-22 + +Announcements: + - Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6). + ## 3.9.3 Released 2017-06-16 diff --git a/package.json b/package.json index c32459f0..10d11d0d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.3", + "version": "3.9.4", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.55.5", + "camshaft": "0.55.6", "cartodb-psql": "0.8.0", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/yarn.lock b/yarn.lock index 3bae7d4a..e0ff12ef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -53,8 +53,8 @@ ap@~0.2.0: resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" aproba@^1.0.3: - version "1.1.1" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" + version "1.1.2" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" are-we-there-yet@~1.1.2: version "1.1.4" @@ -109,9 +109,9 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -balanced-match@^0.4.1: - version "0.4.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -150,21 +150,17 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.0.0: - version "1.1.7" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" +brace-expansion@^1.1.7: + version "1.1.8" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" dependencies: - balanced-match "^0.4.1" + balanced-match "^1.0.0" concat-map "0.0.1" browser-stdout@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" -buffer-shims@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" - buffer-writer@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" @@ -198,9 +194,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.55.5: - version "0.55.5" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.5.tgz#6831f74022b06e12ddab8e00953c7cc859598ac4" +camshaft@0.55.6: + version "0.55.6" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.6.tgz#11af28051c3b911fb023ae1cafb165bbd040f174" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -209,7 +205,7 @@ camshaft@0.55.5: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -227,15 +223,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -421,10 +417,10 @@ debug@2.6.0: ms "0.7.2" debug@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" + version "1.0.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" dependencies: - ms "0.6.2" + ms "2.0.0" decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" @@ -712,7 +708,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: +generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -724,6 +720,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" +generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.6" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -792,8 +792,8 @@ growl@1.9.2: resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: - version "4.0.8" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" + version "4.0.10" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -899,7 +899,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1: +inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -991,12 +991,6 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" -jodid25519@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" - dependencies: - jsbn "~0.1.0" - js-base64@^2.1.9: version "2.1.9" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" @@ -1253,10 +1247,10 @@ mime@~1.2.11: resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: - brace-expansion "^1.0.0" + brace-expansion "^1.1.7" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1277,8 +1271,8 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi minimist "0.0.8" mocha@~3.4.1: - version "3.4.1" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.1.tgz#a3802b4aa381934cacb38de70cf771621da8f9af" + version "3.4.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" dependencies: browser-stdout "1.3.0" commander "2.9.0" @@ -1296,10 +1290,6 @@ moment@^2.10.6: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" -ms@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.6.2.tgz#d89c2124c6fdc1353d65a8b77bf1aac4b193708c" - ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" @@ -1308,6 +1298,10 @@ ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1348,8 +1342,8 @@ nock@~2.11.0: propagate "0.3.x" node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: - version "0.6.34" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" + version "0.6.36" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1404,10 +1398,14 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1508,23 +1506,23 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.7.1" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" + version "1.8.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" dependencies: generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: - version "1.11.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.11.0.tgz#aae91a82d952b633bb88d006350a166daaf6ea90" + version "1.12.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" dependencies: ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" - postgres-interval "~1.0.0" + postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +"pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1537,8 +1535,8 @@ pg@cartodb/node-postgres#6.1.2-cdb1: semver "4.3.2" pgpass@1.x: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.1.tgz#0de8b5bef993295d90a7e17d976f568dcd25d49f" + version "1.0.2" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" dependencies: split "^1.0.0" @@ -1601,9 +1599,9 @@ postgres-date@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" -postgres-interval@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.0.2.tgz#7261438d862b412921c6fdb7617668424b73a6ed" +postgres-interval@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.0.tgz#1031e7bac34564132862adc9eb6c6d2f3aa75bb4" dependencies: xtend "^4.0.0" @@ -1708,14 +1706,14 @@ readable-stream@1.1, readable-stream@~1.1.9: string_decoder "~0.10.x" readable-stream@^2.0.6, readable-stream@^2.1.4: - version "2.2.9" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" + version "2.3.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.2.tgz#5a04df05e4f57fe3f0dc68fdd11dc5c97c7e6f4d" dependencies: - buffer-shims "~1.0.0" core-util-is "~1.0.0" - inherits "~2.0.1" + inherits "~2.0.3" isarray "~1.0.0" process-nextick-args "~1.0.6" + safe-buffer "~5.1.0" string_decoder "~1.0.0" util-deprecate "~1.0.1" @@ -1745,7 +1743,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1768,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1827,7 +1825,11 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1: +safe-buffer@^5.0.1, safe-buffer@~5.1.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" + +safe-buffer@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" @@ -1981,8 +1983,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" + version "1.13.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -1991,7 +1993,6 @@ sshpk@^1.7.0: optionalDependencies: bcrypt-pbkdf "^1.0.0" ecc-jsbn "~0.1.1" - jodid25519 "^1.0.0" jsbn "~0.1.0" tweetnacl "~0.14.0" @@ -2030,10 +2031,10 @@ string_decoder@~0.10.x: resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" string_decoder@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" + version "1.0.2" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.2.tgz#b29e1f4e1125fa97a10382b8a533737b7491e179" dependencies: - buffer-shims "~1.0.0" + safe-buffer "~5.0.1" stringstream@~0.0.4: version "0.0.5" @@ -2107,7 +2108,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2115,7 +2116,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2193,8 +2194,8 @@ type-is@~1.6.10, type-is@~1.6.6: mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.26" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.26.tgz#3a1db8ae0a0aba7f92e1ddadadbd0293d549f90e" + version "2.8.29" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -2234,8 +2235,8 @@ utils-merge@1.0.0: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" uuid@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" validate-npm-package-license@^3.0.1: version "3.0.1" From 2dae09c35b43b251bffc3c641c312010fee08c71 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 22 Jun 2017 18:11:58 +0200 Subject: [PATCH 33/60] Fix yarn.lock and update HOWTO_RELEASE In previous commit I updated all the dependencies with the newest versions when I really intended to update just one. After asking the node gurus we agreed on changing the docs to reflect a less aggressive strategy for dependency upgrades. --- HOWTO_RELEASE | 2 +- yarn.lock | 147 +++++++++++++++++++++++++------------------------- 2 files changed, 74 insertions(+), 75 deletions(-) diff --git a/HOWTO_RELEASE b/HOWTO_RELEASE index be552a95..a2927b60 100644 --- a/HOWTO_RELEASE +++ b/HOWTO_RELEASE @@ -1,7 +1,7 @@ 1. Test (make clean all check), fix if broken before proceeding 2. Ensure proper version in package.json 3. Ensure NEWS section exists for the new version, review it, add release date -4. Recreate yarn.lock with: `yarn upgrade` +4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}` 5. Commit package.json, yarn.lock, NEWS 6. git tag -a Major.Minor.Patch # use NEWS section as content 7. Stub NEWS/package for next version diff --git a/yarn.lock b/yarn.lock index e0ff12ef..8409e6e1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -53,8 +53,8 @@ ap@~0.2.0: resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" aproba@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" + version "1.1.1" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" are-we-there-yet@~1.1.2: version "1.1.4" @@ -109,9 +109,9 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" +balanced-match@^0.4.1: + version "0.4.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -150,17 +150,21 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.1.7: - version "1.1.8" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" +brace-expansion@^1.0.0: + version "1.1.7" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" dependencies: - balanced-match "^1.0.0" + balanced-match "^0.4.1" concat-map "0.0.1" browser-stdout@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" +buffer-shims@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" + buffer-writer@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" @@ -205,7 +209,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +227,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -417,10 +421,10 @@ debug@2.6.0: ms "0.7.2" debug@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" + version "1.0.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.4.tgz#5b9c256bd54b6ec02283176fa8a0ede6d154cbf8" dependencies: - ms "2.0.0" + ms "0.6.2" decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" @@ -708,7 +712,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3: +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -720,10 +724,6 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -792,8 +792,8 @@ growl@1.9.2: resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" handlebars@^4.0.1: - version "4.0.10" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" + version "4.0.8" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -899,7 +899,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@~2.0.0, inherits@~2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -991,6 +991,12 @@ istanbul@~0.4.3: which "^1.1.1" wordwrap "^1.0.0" +jodid25519@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967" + dependencies: + jsbn "~0.1.0" + js-base64@^2.1.9: version "2.1.9" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" @@ -1247,10 +1253,10 @@ mime@~1.2.11: resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + version "3.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" dependencies: - brace-expansion "^1.1.7" + brace-expansion "^1.0.0" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1271,8 +1277,8 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi minimist "0.0.8" mocha@~3.4.1: - version "3.4.2" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" + version "3.4.1" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.1.tgz#a3802b4aa381934cacb38de70cf771621da8f9af" dependencies: browser-stdout "1.3.0" commander "2.9.0" @@ -1290,6 +1296,10 @@ moment@^2.10.6: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" +ms@0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.6.2.tgz#d89c2124c6fdc1353d65a8b77bf1aac4b193708c" + ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" @@ -1298,10 +1308,6 @@ ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" -ms@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1342,8 +1348,8 @@ nock@~2.11.0: propagate "0.3.x" node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: - version "0.6.36" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" + version "0.6.34" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1398,14 +1404,10 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0: +object-assign@4.1.0, object-assign@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1506,23 +1508,23 @@ pg-connection-string@0.1.3: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" pg-pool@1.*: - version "1.8.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" + version "1.7.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.7.1.tgz#421105cb7469979dcc48d6fc4fe3fe4659437437" dependencies: generic-pool "2.4.3" object-assign "4.1.0" pg-types@1.*: - version "1.12.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" + version "1.11.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.11.0.tgz#aae91a82d952b633bb88d006350a166daaf6ea90" dependencies: ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" - postgres-interval "^1.1.0" + postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1535,8 +1537,8 @@ pg-types@1.*: semver "4.3.2" pgpass@1.x: - version "1.0.2" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" + version "1.0.1" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.1.tgz#0de8b5bef993295d90a7e17d976f568dcd25d49f" dependencies: split "^1.0.0" @@ -1599,9 +1601,9 @@ postgres-date@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" -postgres-interval@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.0.tgz#1031e7bac34564132862adc9eb6c6d2f3aa75bb4" +postgres-interval@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.0.2.tgz#7261438d862b412921c6fdb7617668424b73a6ed" dependencies: xtend "^4.0.0" @@ -1706,14 +1708,14 @@ readable-stream@1.1, readable-stream@~1.1.9: string_decoder "~0.10.x" readable-stream@^2.0.6, readable-stream@^2.1.4: - version "2.3.2" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.2.tgz#5a04df05e4f57fe3f0dc68fdd11dc5c97c7e6f4d" + version "2.2.9" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" dependencies: + buffer-shims "~1.0.0" core-util-is "~1.0.0" - inherits "~2.0.3" + inherits "~2.0.1" isarray "~1.0.0" process-nextick-args "~1.0.6" - safe-buffer "~5.1.0" string_decoder "~1.0.0" util-deprecate "~1.0.1" @@ -1743,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1768,7 +1770,7 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.81.0: +request@^2.69.0, request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1825,11 +1827,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@~5.1.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" - -safe-buffer@~5.0.1: +safe-buffer@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" @@ -1983,8 +1981,8 @@ srs@1.x: gdal "~0.9.2" sshpk@^1.7.0: - version "1.13.1" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" + version "1.13.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c" dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -1993,6 +1991,7 @@ sshpk@^1.7.0: optionalDependencies: bcrypt-pbkdf "^1.0.0" ecc-jsbn "~0.1.1" + jodid25519 "^1.0.0" jsbn "~0.1.0" tweetnacl "~0.14.0" @@ -2031,10 +2030,10 @@ string_decoder@~0.10.x: resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" string_decoder@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.2.tgz#b29e1f4e1125fa97a10382b8a533737b7491e179" + version "1.0.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" dependencies: - safe-buffer "~5.0.1" + buffer-shims "~1.0.0" stringstream@~0.0.4: version "0.0.5" @@ -2108,7 +2107,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2116,7 +2115,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2194,8 +2193,8 @@ type-is@~1.6.10, type-is@~1.6.6: mime-types "~2.1.15" uglify-js@^2.6: - version "2.8.29" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" + version "2.8.26" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.26.tgz#3a1db8ae0a0aba7f92e1ddadadbd0293d549f90e" dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -2235,8 +2234,8 @@ utils-merge@1.0.0: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" uuid@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" + version "3.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" validate-npm-package-license@^3.0.1: version "3.0.1" From 15b88c6a67f023e831927dd81d5535d206aff532 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 22 Jun 2017 18:26:33 +0200 Subject: [PATCH 34/60] Stub next version --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 9f334355..5965ae8f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 3.9.5 +Released 2017-mm-dd + ## 3.9.4 Released 2017-06-22 diff --git a/package.json b/package.json index 10d11d0d..404524c1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.4", + "version": "3.9.5", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 32274e66fdb7aa3668af59ec2de2d7eb23d3bbf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 23 Jun 2017 12:24:22 +0200 Subject: [PATCH 35/60] Dataview formula: count infinities and nans as we do with nulls --- lib/cartodb/models/dataview/formula.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 2272089c..50869762 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -14,7 +14,7 @@ var formulaQueryTpl = dot.template([ ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', + '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', From 166e29e8ce9d95aac099df5282ff750d6314e077 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 23 Jun 2017 16:53:16 +0200 Subject: [PATCH 36/60] Forward queries parameter from overview dataviews to base dataviews --- lib/cartodb/models/dataview/overviews/aggregation.js | 2 +- lib/cartodb/models/dataview/overviews/base.js | 5 +++-- lib/cartodb/models/dataview/overviews/formula.js | 4 ++-- lib/cartodb/models/dataview/overviews/histogram.js | 2 +- lib/cartodb/models/dataview/overviews/list.js | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 730ba822..5df092f4 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -90,7 +90,7 @@ var aggregationQueryTpl = dot.template([ var CATEGORIES_LIMIT = 6; function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; this.queries = queries; diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index 1425e2d1..38b2c119 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -1,14 +1,15 @@ var _ = require('underscore'); var BaseDataview = require('../base'); -function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { +function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) { this.BaseDataview = BaseDataview; this.query = query; this.queryOptions = queryOptions; this.queryRewriter = queryRewriter; this.queryRewriteData = queryRewriteData; this.options = options; - this.baseDataview = new this.BaseDataview(this.query, this.queryOptions); + this.queries = queries; + this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries); } module.exports = BaseOverviewsDataview; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 533dd921..a30bc783 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -50,8 +50,8 @@ var formulaQueryTpls = { ].join('\n')), }; -function Formula(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function Formula(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.column = options.column || '1'; this.operation = options.operation; this._isFloatColumn = null; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 361cc8f2..6674f6a0 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -132,7 +132,7 @@ var histogramQueryTpl = dot.template([ ].join('\n')); function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; this.queries = queries; diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js index 7e3b3161..6ec731f4 100644 --- a/lib/cartodb/models/dataview/overviews/list.js +++ b/lib/cartodb/models/dataview/overviews/list.js @@ -1,8 +1,8 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../list'); -function List(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function List(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); } List.prototype = Object.create(BaseOverviewsDataview.prototype); From b2f3735e95d6c7820d7e1a18b494c1a997711221 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 23 Jun 2017 18:59:51 +0200 Subject: [PATCH 37/60] The formula widget wasn't using the no_filters query for checking column types --- lib/cartodb/models/dataview/formula.js | 5 +++-- lib/cartodb/models/dataview/overviews/formula.js | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 50869762..7ec356b7 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -41,7 +41,7 @@ var TYPE = 'formula'; } } */ -function Formula(query, options) { +function Formula(query, options, queries) { if (!_.isString(options.operation)) { throw new Error('Formula expects `operation` in widget options'); } @@ -57,6 +57,7 @@ function Formula(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column || '1'; this.operation = options.operation; this._isFloatColumn = null; @@ -77,7 +78,7 @@ Formula.prototype.sql = function(psql, override, callback) { if (this._isFloatColumn === null) { this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.query, function (err, type) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { if (!err && !!type) { self._isFloatColumn = type.float; } diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index a30bc783..64d612c9 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -55,6 +55,7 @@ function Formula(query, options, queryRewriter, queryRewriteData, params, querie this.column = options.column || '1'; this.operation = options.operation; this._isFloatColumn = null; + this.queries = queries; } Formula.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -70,7 +71,7 @@ Formula.prototype.sql = function (psql, override, callback) { // supported formula for use with overviews if (this._isFloatColumn === null) { this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.query, function (err, type) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { if (!err && !!type) { self._isFloatColumn = type.float; } From 3a442bea44b3a03a5ed384f594a7585c27b2a4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 17:06:24 +0200 Subject: [PATCH 38/60] Update NEWS --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 5965ae8f..b96a7b94 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ ## 3.9.5 Released 2017-mm-dd + - Dataviews: support special numeric values (±Infinity, NaN) #700 + + ## 3.9.4 Released 2017-06-22 From 2f1cacdfc795f390d05bf4441097185f3abaf1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 17:07:52 +0200 Subject: [PATCH 39/60] Release 3.9.5 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b96a7b94..64c33d4d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.9.5 -Released 2017-mm-dd +Released 2017-06-27 - Dataviews: support special numeric values (±Infinity, NaN) #700 From f507f7a74b2e9b6c3b5a52d60bc56b57b567d7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 17:18:08 +0200 Subject: [PATCH 40/60] Stub next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 64c33d4d..454b9142 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.6 +Released 2017-mm-dd + + ## 3.9.5 Released 2017-06-27 diff --git a/package.json b/package.json index 404524c1..463848c5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.5", + "version": "3.9.6", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From f7a763b6378c4dc9ce1d334f11582f84ec20e1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:09:17 +0200 Subject: [PATCH 41/60] widgets: add aggregation for search results --- lib/cartodb/models/dataview/aggregation.js | 3 ++- package.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index c15f0506..2fb76600 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -253,6 +253,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { var self = this; var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); + var _value = this.aggregation !== 'count' && this.aggregationColumn ? this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ @@ -265,7 +266,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { _searchFiltered: filterCategoriesQueryTpl({ _query: this.query, _column: this.column, - _value: 'count(1)', + _value: _value, _userQuery: _userQuery }) }); diff --git a/package.json b/package.json index c32459f0..f613df30 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "contributors": [ "Simon Tokumine ", "Javi Santana ", - "Sandro Santilli " + "Sandro Santilli ", + "Carlos Matallín " ], "dependencies": { "body-parser": "~1.14.0", From 7fa154c06286d8ae9372f18119a7da6d2fa3b503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:38:15 +0200 Subject: [PATCH 42/60] widgets: add aggregation for search results, specs --- test/acceptance/widgets/ported/aggregation.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/acceptance/widgets/ported/aggregation.js b/test/acceptance/widgets/ported/aggregation.js index 9329f039..62c998a5 100644 --- a/test/acceptance/widgets/ported/aggregation.js +++ b/test/acceptance/widgets/ported/aggregation.js @@ -322,6 +322,25 @@ describe('widgets', function() { }); }); }); + + [adm0name].forEach(function(userQuery) { + it('should search with sum aggregation: ' + userQuery, function(done) { + this.testClient = new TestClient(aggregationSumMapConfig); + this.testClient.widgetSearch('adm0name', userQuery, function (err, res, searchResult) { + assert.ok(!err, err); + assert.ok(searchResult); + assert.equal(searchResult.type, 'aggregation'); + + assert.equal(searchResult.categories.length, 1); + assert.deepEqual( + searchResult.categories, + [{ category:"Argentina", value:28015640 }] + ); + + done(); + }); + }); + }); }); }); From dd934a39139ea07d76709000b827b99143c38f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:44:32 +0200 Subject: [PATCH 43/60] linter --- Makefile | 2 +- lib/cartodb/models/dataview/aggregation.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1913b9e8..a2910a0f 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ config/environments/test.js: config.status--test TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js") TEST_SUITE_UNIT := $(shell find test/unit -name "*.js") TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js") -TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js") +TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance/widgets/ported -name "*.js") test: config/environments/test.js @echo "***tests***" diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 2fb76600..d7710f4f 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -253,7 +253,8 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { var self = this; var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); - var _value = this.aggregation !== 'count' && this.aggregationColumn ? this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; + var _value = this.aggregation !== 'count' && this.aggregationColumn ? + this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ From b1c9dd537e0f07486f3d6a6c8d51fca493cc3a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 7 Jul 2017 17:47:28 +0200 Subject: [PATCH 44/60] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a2910a0f..1913b9e8 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ config/environments/test.js: config.status--test TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js") TEST_SUITE_UNIT := $(shell find test/unit -name "*.js") TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js") -TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance/widgets/ported -name "*.js") +TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js") test: config/environments/test.js @echo "***tests***" From e36266a80f7d2736bd94e71f334aeba1c90a777b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 10 Jul 2017 12:29:25 +0200 Subject: [PATCH 45/60] Added test to check all aggregation operations work as expected when searching dataviews --- test/acceptance/dataviews/aggregation.js | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 259cd2af..c4b40d44 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -145,4 +145,33 @@ describe('aggregations happy cases', function() { }); }); }); + + var widgetSearchExpects = { + 'count': [ { category: 'other_a', value: 3 } ], + 'sum': [ { category: 'other_a', value: 6 } ], + 'avg': [ { category: 'other_a', value: 2 } ], + 'max': [ { category: 'other_a', value: 3 } ], + 'min': [ { category: 'other_a', value: 1 } ] + }; + + Object.keys(operations_and_values).forEach(function (operation) { + var description = 'should search OTHER category using "' + operation + '"'; + + it(description, function (done) { + this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val')); + this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) { + assert.ifError(err); + + assert.ok(searchResult); + assert.equal(searchResult.type, 'aggregation'); + + assert.equal(searchResult.categories.length, 1); + assert.deepEqual( + searchResult.categories, + widgetSearchExpects[operation] + ); + done(); + }); + }); + }); }); From 5b8ecd3df0a2bc46bf8b5619f3552cd8e9361693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Jul 2017 09:53:12 +0200 Subject: [PATCH 46/60] Release 3.9.6 --- NEWS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 454b9142..b59924bc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,9 @@ # Changelog ## 3.9.6 -Released 2017-mm-dd +Released 2017-07-11 + + - Dataviews: support for aggregation in search results #708 ## 3.9.5 From a35b1e3e8656e9b354e6f092c1099f8e674fe0ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 11 Jul 2017 09:59:30 +0200 Subject: [PATCH 47/60] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b59924bc..dd6b97f4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.7 +Released 2017-mm-dd + + ## 3.9.6 Released 2017-07-11 diff --git a/package.json b/package.json index 4ed05477..5d6a40df 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.6", + "version": "3.9.7", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From eccc3597aa68113cf9d69c24caac7f63af8a8f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 17 Jul 2017 19:43:59 +0200 Subject: [PATCH 48/60] Respond with 204 when vector tile is empty --- lib/cartodb/controllers/base.js | 4 +++ test/acceptance/mvt.js | 58 +++++++++++++++++++++++++++++++++ test/support/test-client.js | 16 ++++++--- 3 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 test/acceptance/mvt.js diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index ab6587c2..7cdd0713 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -201,6 +201,10 @@ BaseController.prototype.sendError = function(req, res, err, label) { var statusCode = findStatusCode(err); + if (err.message === 'Tile does not exist' && req.params.format === 'mvt') { + statusCode = 204; + } + debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack); // If a callback was requested, force status to 200 diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js new file mode 100644 index 00000000..17653d1d --- /dev/null +++ b/test/acceptance/mvt.js @@ -0,0 +1,58 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); + +function createMapConfig (sql) { + sql = sql || [ + 'select', + ' *', + 'from', + ' populated_places_simple_reduced', + ].join('\n'); + + return { + version: '1.6.0', + layers: [{ + type: "cartodb", + options: { + sql: sql, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + interactivity: 'cartodb_id' + } + }] + }; +} + + +describe('mvt', function () { + const testCases = [ + { + desc: 'should get empty mvt with code 204 (no content)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + mapConfig: createMapConfig('select 1 as cartodb_id, null::geometry as the_geom_webmercator') + } + ]; + + testCases.forEach(function (test) { + it(test.desc, done => { + const testClient = new TestClient(test.mapConfig, 1234); + const { z, x, y } = test.coords; + const options = { + format: test.format, + status: 204 + }; + + testClient.getTile(z, x, y, options, (err, res) => { + assert.ifError(err); + + assert.ifError(err); + assert.equal(res.statusCode, 204); + assert.equal(res.body, ''); + testClient.drain(done); + }); + }); + }); +}); \ No newline at end of file diff --git a/test/support/test-client.js b/test/support/test-client.js index 659d6686..72a6eca9 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -525,7 +525,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { }; var expectedResponse = { - status: 200, + status: params.status || 200, headers: { 'Content-Type': 'application/json; charset=utf-8' } @@ -542,7 +542,12 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { if (isMvt) { request.encoding = 'binary'; - expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; + + if (expectedResponse.status === 200) { + expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; + } else if (expectedResponse.status === 204) { + expectedResponse.headers['Content-Type'] = undefined; + } } var isGeojson = format.match(/geojson$/); @@ -561,15 +566,16 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { assert.response(server, request, expectedResponse, function(res, err) { assert.ifError(err); - var obj; if (isPng) { obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); } else if (isMvt) { - obj = new mapnik.VectorTile(z, x, y); - obj.setDataSync(new Buffer(res.body, 'binary')); + if (res.body) { + obj = new mapnik.VectorTile(z, x, y); + obj.setDataSync(new Buffer(res.body, 'binary')); + } } else { obj = JSON.parse(res.body); From ff1399625568432c2da6eb4e5a54ca42ddb53006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 10:44:27 +0200 Subject: [PATCH 49/60] Add test to check that mvt returns 200 when tile has data --- test/acceptance/mvt.js | 34 +++++++++++++++------------------- test/support/test-client.js | 5 +++++ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 17653d1d..db75bb16 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -1,16 +1,9 @@ require('../support/test_helper'); -var assert = require('../support/assert'); -var TestClient = require('../support/test-client'); - -function createMapConfig (sql) { - sql = sql || [ - 'select', - ' *', - 'from', - ' populated_places_simple_reduced', - ].join('\n'); +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); +function createMapConfig (sql = TestClient.SQL.ONE_POINT) { return { version: '1.6.0', layers: [{ @@ -25,14 +18,21 @@ function createMapConfig (sql) { }; } - describe('mvt', function () { const testCases = [ { desc: 'should get empty mvt with code 204 (no content)', coords: { z: 0, x: 0, y: 0 }, format: 'mvt', - mapConfig: createMapConfig('select 1 as cartodb_id, null::geometry as the_geom_webmercator') + status: 204, + mapConfig: createMapConfig(TestClient.SQL.EMPTY) + }, + { + desc: 'should get mvt tile with code 200 (ok)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + status: 200, + mapConfig: createMapConfig() } ]; @@ -40,17 +40,13 @@ describe('mvt', function () { it(test.desc, done => { const testClient = new TestClient(test.mapConfig, 1234); const { z, x, y } = test.coords; - const options = { - format: test.format, - status: 204 - }; + const { format, status } = test; - testClient.getTile(z, x, y, options, (err, res) => { + testClient.getTile(z, x, y, { format, status }, (err, res) => { assert.ifError(err); assert.ifError(err); - assert.equal(res.statusCode, 204); - assert.equal(res.body, ''); + assert.equal(res.statusCode, test.status); testClient.drain(done); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 72a6eca9..5d0eddff 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -75,6 +75,11 @@ module.exports.CARTOCSS = { ].join('\n') }; +module.exports.SQL = { + EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator', + ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' +} + TestClient.prototype.getWidget = function(widgetName, params, callback) { var self = this; From 0aab434f13fb019e9c23059309fa9941afa6c39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 10:52:24 +0200 Subject: [PATCH 50/60] Remove duplicated assertion --- test/acceptance/mvt.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index db75bb16..6510541b 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -45,7 +45,6 @@ describe('mvt', function () { testClient.getTile(z, x, y, { format, status }, (err, res) => { assert.ifError(err); - assert.ifError(err); assert.equal(res.statusCode, test.status); testClient.drain(done); }); From 446e2d08020fa51c1077a137b3c3bd6e9d94bf9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 11:05:45 +0200 Subject: [PATCH 51/60] Add empty line at the end of file --- test/acceptance/mvt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 6510541b..b1ea4455 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -50,4 +50,4 @@ describe('mvt', function () { }); }); }); -}); \ No newline at end of file +}); From f306c26da61196b7903f94c4ef93a04a6a7fb69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 11:08:39 +0200 Subject: [PATCH 52/60] Update News --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index dd6b97f4..fbce5c7d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.9.7 Released 2017-mm-dd + - Respond with 204 (No content) when vector tile has no data #712 + ## 3.9.6 Released 2017-07-11 From a17916488b4760e0784612400f4d2591ef8867f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 18 Jul 2017 17:40:36 +0200 Subject: [PATCH 53/60] Update config to get the right key to enable stats metadate after layergroup creation --- config/environments/development.js.example | 3 +-- config/environments/production.js.example | 2 +- config/environments/staging.js.example | 2 +- config/environments/test.js.example | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 9bc4157b..48d03592 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -324,8 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true - + layerStats: true } }; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index b305deef..ba7fec6f 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -324,7 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: false + layerStats: false } }; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index fa3c7cb6..1792779c 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -324,7 +324,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true + layerStats: true } }; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 465b1774..374467c0 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -318,7 +318,7 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerMetadata: true + layerStats: true } }; From cc8a1df3885f875de772a699bc0bd2da8b4c48f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 12:11:09 +0200 Subject: [PATCH 54/60] Bump turbo-carto version to 0.19.2 --- NEWS.md | 4 ++++ package.json | 2 +- yarn.lock | 10 +++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index fbce5c7d..115c4aa1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,8 +3,12 @@ ## 3.9.7 Released 2017-mm-dd +Bug fixes: - Respond with 204 (No content) when vector tile has no data #712 +Announcements: + - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1) + ## 3.9.6 Released 2017-07-11 diff --git a/package.json b/package.json index 5d6a40df..c73280ec 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "request": "~2.79.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.19.1", + "turbo-carto": "0.19.2", "underscore": "~1.6.0", "windshaft": "3.2.1", "yargs": "~5.0.0" diff --git a/yarn.lock b/yarn.lock index 8409e6e1..a5538d97 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1745,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1770,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -2156,9 +2156,9 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@0.19.1: - version "0.19.1" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.1.tgz#c32af073936a4e8f197dfea918e7441c949d7865" +turbo-carto@^0.19.2: + version "0.19.2" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" From 20003c49cec48ab0c26c08fb2d8904738493f8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 14:53:20 +0200 Subject: [PATCH 55/60] Release 3.9.7 --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 115c4aa1..a0a0aa0a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,13 +1,13 @@ # Changelog ## 3.9.7 -Released 2017-mm-dd +Released 2017-07-20 Bug fixes: - Respond with 204 (No content) when vector tile has no data #712 Announcements: - - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1) + - Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2) ## 3.9.6 From fc3a959da1bbb301ea840cef5bf0880e97ee0780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 15:00:39 +0200 Subject: [PATCH 56/60] Stub next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index a0a0aa0a..cc74a354 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.8 +Released 2017-mm-dd + + ## 3.9.7 Released 2017-07-20 diff --git a/package.json b/package.json index c73280ec..f3fd81a7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.7", + "version": "3.9.8", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 42d05f29ee6b5dae6cf27dcae89bc8a1da716fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 20 Jul 2017 19:47:01 +0200 Subject: [PATCH 57/60] Bump windshaft version to 3.2.2 --- package.json | 2 +- yarn.lock | 66 ++++++++++++++++++++++++---------------------------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/package.json b/package.json index f3fd81a7..d15c9667 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.2", "underscore": "~1.6.0", - "windshaft": "3.2.1", + "windshaft": "3.2.2", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index a5538d97..02d88ef6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -53,8 +53,8 @@ ap@~0.2.0: resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" aproba@^1.0.3: - version "1.1.1" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" + version "1.1.2" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" are-we-there-yet@~1.1.2: version "1.1.4" @@ -161,10 +161,6 @@ browser-stdout@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" -buffer-shims@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" - buffer-writer@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" @@ -209,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -235,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -899,7 +895,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1: +inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -1348,8 +1344,8 @@ nock@~2.11.0: propagate "0.3.x" node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: - version "0.6.34" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" + version "0.6.36" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" dependencies: mkdirp "^0.5.1" nopt "^4.0.1" @@ -1388,8 +1384,8 @@ normalize-package-data@^2.3.2: validate-npm-package-license "^3.0.1" npmlog@^4.0.2: - version "4.1.0" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.0.tgz#dc59bee85f64f00ed424efb2af0783df25d1c0b5" + version "4.1.2" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" dependencies: are-we-there-yet "~1.1.2" console-control-strings "~1.1.0" @@ -1708,15 +1704,15 @@ readable-stream@1.1, readable-stream@~1.1.9: string_decoder "~0.10.x" readable-stream@^2.0.6, readable-stream@^2.1.4: - version "2.2.9" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" + version "2.3.3" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" dependencies: - buffer-shims "~1.0.0" core-util-is "~1.0.0" - inherits "~2.0.1" + inherits "~2.0.3" isarray "~1.0.0" process-nextick-args "~1.0.6" - string_decoder "~1.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.0.3" util-deprecate "~1.0.1" readable-stream@~1.0.2: @@ -1827,9 +1823,9 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" +safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" safe-json-stringify@~1: version "1.0.4" @@ -2029,11 +2025,11 @@ string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" -string_decoder@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" +string_decoder@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" dependencies: - buffer-shims "~1.0.0" + safe-buffer "~5.1.0" stringstream@~0.0.4: version "0.0.5" @@ -2107,15 +2103,15 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: - version "2.3.1-cdb2" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3": + version "2.3.1-cdb3" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2156,7 +2152,7 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@^0.19.2: +turbo-carto@0.19.2: version "0.19.2" resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd" dependencies: @@ -2278,9 +2274,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.1.tgz#50a3afa6562315dd9e65e411660970e118f36c19" +windshaft@3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.2.tgz#7afb9d8fd8bba1bf02d39c06e8bbe5a451aad953" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2297,7 +2293,7 @@ windshaft@3.2.1: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb2 + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb3 tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2 torque.js "~2.11.0" underscore "~1.6.0" From 106b9a64b23b0090c8ae7222d168de1a4c4bf591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 21 Jul 2017 10:39:33 +0200 Subject: [PATCH 58/60] Update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index cc74a354..6a591fff 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.9.8 Released 2017-mm-dd + - Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2). + ## 3.9.7 Released 2017-07-20 From 35b9448e9a18f976459a762fee5ad0733b99c0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 21 Jul 2017 10:58:37 +0200 Subject: [PATCH 59/60] Release 3.9.8 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6a591fff..87124867 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.9.8 -Released 2017-mm-dd +Released 2017-07-21 - Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2). From 52da3bfa55f8fc07d8215f6bd3b6191954b2c973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 21 Jul 2017 11:05:51 +0200 Subject: [PATCH 60/60] Stub next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 87124867..a3228af1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 3.9.9 +Released 2017-mm-dd + + ## 3.9.8 Released 2017-07-21 diff --git a/package.json b/package.json index d15c9667..08dc20cf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.8", + "version": "3.9.9", "description": "A map tile server for CartoDB", "keywords": [ "cartodb"