cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Formula dataview: support special values only if column is a float column

Browse Source
This commit is contained in:
Daniel García Aubert 2017-06-15 16:31:24 +02:00
parent 7d0af4e259
commit 443c1100d7
3 changed files with 102 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
lib/cartodb/models/dataview/formula.js
View File

@ -5,25 +5,25 @@ var debug = require('debug')('windshaft:widget:formula');
var dot = require('dot');
dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var formulaQueryTpl = dot.template([
'SELECT',
' {{=it._operation}}({{=it._column}}) AS result,',
' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
' {{?it._operation!==\'count\'}}',
' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count',
' {{?}}',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._operation!==\'count\'}}',
'WHERE',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float',
'{{?}}'
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n'));
var VALID_OPERATIONS = {
@ -34,6 +34,11 @@ var VALID_OPERATIONS = {
max: true
};
var FLOAT_OIDS = {
700: true,
701: true
};
var TYPE = 'formula';
/**
@ -63,6 +68,7 @@ function Formula(query, options) {
this.query = query;
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
}
Formula.prototype = new BaseWidget();
@ -71,13 +77,36 @@ Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var _query = this.query;
var columnTypeQuery = columnTypeQueryTpl({
column: this.column, query: _query
});
if (this._isFloatColumn === null) {
var readOnlyTransaction = true;
psql.query(columnTypeQuery, function(err, result) {
self._isFloatColumn = false;
if (!err && !!result.rows[0]) {
var pgType = result.rows[0].pg_typeof;
if (FLOAT_OIDS.hasOwnProperty(pgType)) {
self._isFloatColumn = true;
}
}
self.sql(psql, override, callback);
}, readOnlyTransaction);
return null;
}
var formulaSql = formulaQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_operation: this.operation,
_column: this.column

68
lib/cartodb/models/dataview/overviews/formula.js
View File

@ -1,47 +1,61 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overview');
var dot = require('dot');
dot.templateSettings.strip = false;
var FLOAT_OIDS = {
700: true,
701: true
};
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var formulaQueryTpls = {
'count': dot.template([
'SELECT',
'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'WHERE',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float'
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'WHERE',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float'
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
};
@ -49,6 +63,7 @@ function Formula(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
}
Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
@ -57,20 +72,45 @@ Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) {
var self = this;
var formulaQueryTpl = formulaQueryTpls[this.operation];
if ( formulaQueryTpl ) {
// supported formula for use with overviews
var columnTypeQuery = columnTypeQueryTpl({
column: this.column, query: this.query
});
if (this._isFloatColumn === null) {
var readOnlyTransaction = true;
psql.query(columnTypeQuery, function(err, result) {
self._isFloatColumn = false;
if (!err && !!result.rows[0]) {
var pgType = result.rows[0].pg_typeof;
if (FLOAT_OIDS.hasOwnProperty(pgType)) {
self._isFloatColumn = true;
}
}
self.sql(psql, override, callback);
}, readOnlyTransaction);
return null;
}
var formulaSql = formulaQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: this.rewrittenQuery(this.query),
_operation: this.operation,
_column: this.column
});
callback = callback || override;
debug(formulaSql);
return callback(null, formulaSql);
}
// default behaviour
return this.defaultSql(psql, override, callback);
};

13
test/acceptance/dataviews/overviews.js
View File

@ -303,7 +303,9 @@ describe('dataviews using tables with overviews', function() {
"operation":"count",
"result":5,
"nulls":0,
"type":"formula"
"type":"formula",
"infinities": 0,
"nans": 0
});
testClient.drain(done);
@ -498,7 +500,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"count",
"result":1,
"infinities": 0,
"nans": 0,
"nulls":0,
"type":"formula"
});
testClient.drain(done);
});