From 443c1100d7487b2f2d29a8a023eb3bf0d7675f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 15 Jun 2017 16:31:24 +0200 Subject: [PATCH] Formula dataview: support special values only if column is a float column --- lib/cartodb/models/dataview/formula.js | 45 +++++++++--- .../models/dataview/overviews/formula.js | 68 +++++++++++++++---- test/acceptance/dataviews/overviews.js | 13 +++- 3 files changed, 102 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 04b89835..1d3204e7 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -5,25 +5,25 @@ var debug = require('debug')('windshaft:widget:formula'); var dot = require('dot'); dot.templateSettings.strip = false; +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); + var formulaQueryTpl = dot.template([ 'SELECT', ' {{=it._operation}}({{=it._column}}) AS result,', ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - ' {{?it._operation!==\'count\'}}', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', - ' {{?}}', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._operation!==\'count\'}}', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float', - '{{?}}' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')); var VALID_OPERATIONS = { @@ -34,6 +34,11 @@ var VALID_OPERATIONS = { max: true }; +var FLOAT_OIDS = { + 700: true, + 701: true +}; + var TYPE = 'formula'; /** @@ -63,6 +68,7 @@ function Formula(query, options) { this.query = query; this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = new BaseWidget(); @@ -71,13 +77,36 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } var _query = this.query; + + var columnTypeQuery = columnTypeQueryTpl({ + column: this.column, query: _query + }); + + if (this._isFloatColumn === null) { + var readOnlyTransaction = true; + psql.query(columnTypeQuery, function(err, result) { + self._isFloatColumn = false; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (FLOAT_OIDS.hasOwnProperty(pgType)) { + self._isFloatColumn = true; + } + } + self.sql(psql, override, callback); + }, readOnlyTransaction); + return null; + } + var formulaSql = formulaQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _operation: this.operation, _column: this.column diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index ba98b2c8..a1437c73 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,47 +1,61 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); +var debug = require('debug')('windshaft:widget:formula:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var FLOAT_OIDS = { + 700: true, + 701: true +}; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); + var formulaQueryTpls = { 'count': dot.template([ 'SELECT', 'sum(_feature_count) AS result,', '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula' ].join('\n')), 'sum': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')), 'avg': dot.template([ 'SELECT', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', 'FROM ({{=it._query}}) _cdb_formula', - 'WHERE', + '{{?it._isFloatColumn}}WHERE', ' {{=it._column}} != \'infinity\'::float', 'AND', ' {{=it._column}} != \'-infinity\'::float', 'AND', - ' {{=it._column}} != \'NaN\'::float' + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')), }; @@ -49,6 +63,7 @@ function Formula(query, options, queryRewriter, queryRewriteData, params) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -57,20 +72,45 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; var formulaQueryTpl = formulaQueryTpls[this.operation]; if ( formulaQueryTpl ) { // supported formula for use with overviews + + var columnTypeQuery = columnTypeQueryTpl({ + column: this.column, query: this.query + }); + + if (this._isFloatColumn === null) { + var readOnlyTransaction = true; + psql.query(columnTypeQuery, function(err, result) { + self._isFloatColumn = false; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (FLOAT_OIDS.hasOwnProperty(pgType)) { + self._isFloatColumn = true; + } + } + self.sql(psql, override, callback); + }, readOnlyTransaction); + return null; + } + var formulaSql = formulaQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: this.rewrittenQuery(this.query), _operation: this.operation, _column: this.column }); callback = callback || override; + debug(formulaSql); + return callback(null, formulaSql); } + // default behaviour return this.defaultSql(psql, override, callback); }; diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 4362d7a3..dcff687f 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -303,7 +303,9 @@ describe('dataviews using tables with overviews', function() { "operation":"count", "result":5, "nulls":0, - "type":"formula" + "type":"formula", + "infinities": 0, + "nans": 0 }); testClient.drain(done); @@ -498,7 +500,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":1, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); });