From 668b22628ce4ce5cfbf589ef50ab4868e549c11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 13 Jun 2017 19:01:59 +0200 Subject: [PATCH] Going green: support special float values in aggergation dataview with overviews --- .../models/dataview/overviews/aggregation.js | 63 +++++++++++++++---- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index da63b27f..1b0e2ec6 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,14 +1,38 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../aggregation'); +var debug = require('debug')('windshaft:widget:aggregation:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' {{?it._aggregationColumn}}', + ' WHERE', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float', + ' {{?}}', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' sum(_feature_count) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count,', + ' sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -17,7 +41,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', @@ -43,21 +67,23 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', + ' count, categories_count, nans_count, infinities_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count, nans_count, infinities_count', 'ORDER BY value DESC' ].join('\n')); @@ -84,21 +110,28 @@ Aggregation.prototype.sql = function(psql, override, callback) { } var _query = this.rewrittenQuery(this.query); + var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ "WITH", [ + filteredQueryTpl({ + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn + }), summaryQueryTpl({ _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -120,15 +153,21 @@ Aggregation.prototype.sql = function(psql, override, callback) { aggregationSql = [ "WITH", [ + filteredQueryTpl({ + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn + }), summaryQueryTpl({ _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -147,6 +186,8 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } + debug(aggregationSql); + return callback(null, aggregationSql); };