cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Handle special float values only when aggregation columns is float (overviews)

Browse Source
This commit is contained in:
Daniel García Aubert 2017-06-15 19:22:26 +02:00
parent 115d8fe685
commit 20d7f1a7c5
1 changed files with 34 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
lib/cartodb/models/dataview/overviews/aggregation.js
View File

@ -9,14 +9,12 @@ var filteredQueryTpl = dot.template([
'filtered_source AS (', 'filtered_source AS (',
' SELECT *', ' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source', ' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn}}', ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float', ' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND', ' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float', ' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND', ' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float', ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
' {{?}}',
')' ')'
].join(' \n')); ].join(' \n'));
@ -24,15 +22,15 @@ var summaryQueryTpl = dot.template([
'summary AS (', 'summary AS (',
' SELECT', ' SELECT',
' sum(_feature_count) AS count,', ' sum(_feature_count) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' sum(', ' {{?it._isFloatColumn}},sum(',
' CASE', ' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1', ' THEN 1',
' ELSE 0', ' ELSE 0',
' END', ' END',
' ) AS infinities_count,', ' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls', ' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')' ')'
].join('\n')); ].join('\n'));
@ -68,34 +66,38 @@ var categoriesSummaryCountQueryTpl = dot.template([
var rankedAggregationQueryTpl = dot.template([ var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count, nans_count, infinities_count', ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}', ' WHERE rank < {{=it._limit}}',
'UNION ALL', 'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
' count, categories_count, nans_count, infinities_count', ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}', ' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' 'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n')); ].join('\n'));
var aggregationQueryTpl = dot.template([ var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count', 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC' 'ORDER BY value DESC'
].join('\n')); ].join('\n'));
var CATEGORIES_LIMIT = 6; var CATEGORIES_LIMIT = 6;
function Aggregation(query, options, queryRewriter, queryRewriteData, params) { function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
this.query = query; this.query = query;
this.queries = queries;
this.column = options.column; this.column = options.column;
this.aggregation = options.aggregation; this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn; this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
} }
Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
@ -104,6 +106,8 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation; module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) { Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) { if (!callback) {
callback = override; callback = override;
override = {}; override = {};
@ -112,17 +116,30 @@ Aggregation.prototype.sql = function(psql, override, callback) {
var _query = this.rewrittenQuery(this.query); var _query = this.rewrittenQuery(this.query);
var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var aggregationSql; var aggregationSql;
if (!!override.ownFilter) { if (!!override.ownFilter) {
aggregationSql = [ aggregationSql = [
"WITH", "WITH",
[ [
filteredQueryTpl({ filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregationColumn: _aggregationColumn _aggregationColumn: _aggregationColumn
}), }),
summaryQueryTpl({ summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregationColumn: _aggregationColumn _aggregationColumn: _aggregationColumn
@ -143,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
}) })
].join(',\n'), ].join(',\n'),
aggregationQueryTpl({ aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregation: this.getAggregationSql(), _aggregation: this.getAggregationSql(),
@ -154,11 +172,13 @@ Aggregation.prototype.sql = function(psql, override, callback) {
"WITH", "WITH",
[ [
filteredQueryTpl({ filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregationColumn: _aggregationColumn _aggregationColumn: _aggregationColumn
}), }),
summaryQueryTpl({ summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregationColumn: _aggregationColumn _aggregationColumn: _aggregationColumn
@ -179,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
}) })
].join(',\n'), ].join(',\n'),
rankedAggregationQueryTpl({ rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_limit: CATEGORIES_LIMIT _limit: CATEGORIES_LIMIT