Merge branch 'master' into 691-date-histogram

This commit is contained in:
Daniel García Aubert 2017-08-01 16:07:27 +02:00
commit b2b68ffd5c
26 changed files with 1357 additions and 251 deletions

View File

@ -1,7 +1,7 @@
1. Test (make clean all check), fix if broken before proceeding 1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json 2. Ensure proper version in package.json
3. Ensure NEWS section exists for the new version, review it, add release date 3. Ensure NEWS section exists for the new version, review it, add release date
4. Recreate yarn.lock with: `yarn upgrade` 4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
5. Commit package.json, yarn.lock, NEWS 5. Commit package.json, yarn.lock, NEWS
6. git tag -a Major.Minor.Patch # use NEWS section as content 6. git tag -a Major.Minor.Patch # use NEWS section as content
7. Stub NEWS/package for next version 7. Stub NEWS/package for next version

38
NEWS.md
View File

@ -1,5 +1,43 @@
# Changelog # Changelog
## 3.9.9
Released 2017-mm-dd
## 3.9.8
Released 2017-07-21
- Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2).
## 3.9.7
Released 2017-07-20
Bug fixes:
- Respond with 204 (No content) when vector tile has no data #712
Announcements:
- Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2)
## 3.9.6
Released 2017-07-11
- Dataviews: support for aggregation in search results #708
## 3.9.5
Released 2017-06-27
- Dataviews: support special numeric values (±Infinity, NaN) #700
## 3.9.4
Released 2017-06-22
Announcements:
- Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6).
## 3.9.3 ## 3.9.3
Released 2017-06-16 Released 2017-06-16

View File

@ -324,8 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true, cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer // whether in mapconfig is available stats & metadata for each layer
layerMetadata: true layerStats: true
} }
}; };

View File

@ -324,7 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true, cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer // whether in mapconfig is available stats & metadata for each layer
layerMetadata: false layerStats: false
} }
}; };

View File

@ -324,7 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true, cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer // whether in mapconfig is available stats & metadata for each layer
layerMetadata: true layerStats: true
} }
}; };

View File

@ -318,7 +318,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true, cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer // whether in mapconfig is available stats & metadata for each layer
layerMetadata: true layerStats: true
} }
}; };

View File

@ -193,6 +193,10 @@ BaseController.prototype.sendError = function(req, res, err, label) {
var statusCode = findStatusCode(err); var statusCode = findStatusCode(err);
if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
statusCode = 204;
}
debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack); debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
// If a callback was requested, force status to 200 // If a callback was requested, force status to 200

View File

@ -5,11 +5,32 @@ var debug = require('debug')('windshaft:widget:aggregation');
var dot = require('dot'); var dot = require('dot');
dot.templateSettings.strip = false; dot.templateSettings.strip = false;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var summaryQueryTpl = dot.template([ var summaryQueryTpl = dot.template([
'summary AS (', 'summary AS (',
' SELECT', ' SELECT',
' count(1) AS count,', ' count(1) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' {{?it._isFloatColumn}},sum(',
' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1',
' ELSE 0',
' END',
' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls', ' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')' ')'
].join('\n')); ].join('\n'));
@ -18,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
'categories AS(', 'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all', ' FROM filtered_source',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}', ' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC', ' ORDER BY 2 DESC',
@ -44,22 +65,25 @@ var categoriesSummaryCountQueryTpl = dot.template([
].join('\n')); ].join('\n'));
var rankedAggregationQueryTpl = dot.template([ var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}', ' WHERE rank < {{=it._limit}}',
'UNION ALL', 'UNION ALL',
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,',
' count, categories_count', ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}', ' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count' 'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n')); ].join('\n'));
var aggregationQueryTpl = dot.template([ var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count', ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', 'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC' 'ORDER BY value DESC'
].join('\n')); ].join('\n'));
@ -84,7 +108,7 @@ var TYPE = 'aggregation';
} }
} }
*/ */
function Aggregation(query, options) { function Aggregation(query, options, queries) {
if (!_.isString(options.column)) { if (!_.isString(options.column)) {
throw new Error('Aggregation expects `column` in widget options'); throw new Error('Aggregation expects `column` in widget options');
} }
@ -108,9 +132,11 @@ function Aggregation(query, options) {
BaseWidget.apply(this); BaseWidget.apply(this);
this.query = query; this.query = query;
this.queries = queries;
this.column = options.column; this.column = options.column;
this.aggregation = options.aggregation; this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn; this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
} }
Aggregation.prototype = new BaseWidget(); Aggregation.prototype = new BaseWidget();
@ -119,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation; module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) { Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) { if (!callback) {
callback = override; callback = override;
override = {}; override = {};
} }
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var _query = this.query; var _query = this.query;
var aggregationSql; var aggregationSql;
if (!!override.ownFilter) { if (!!override.ownFilter) {
aggregationSql = [ aggregationSql = [
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
aggregationQueryTpl({ aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregation: this.getAggregationSql(), _aggregation: this.getAggregationSql(),
@ -140,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) {
].join('\n'); ].join('\n');
} else { } else {
aggregationSql = [ aggregationSql = [
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
rankedAggregationQueryTpl({ rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
@ -155,13 +208,21 @@ Aggregation.prototype.sql = function(psql, override, callback) {
return callback(null, aggregationSql); return callback(null, aggregationSql);
}; };
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) {
return [ return [
"WITH", "WITH",
[ [
filteredQueryTpl({
_isFloatColumn: isFloatColumn,
_query: this.query,
_column: this.column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
summaryQueryTpl({ summaryQueryTpl({
_isFloatColumn: isFloatColumn,
_query: query, _query: query,
_column: column _column: column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}), }),
rankedCategoriesQueryTpl({ rankedCategoriesQueryTpl({
_query: query, _query: query,
@ -193,6 +254,8 @@ Aggregation.prototype.format = function(result) {
var categories = []; var categories = [];
var count = 0; var count = 0;
var nulls = 0; var nulls = 0;
var nans = 0;
var infinities = 0;
var minValue = 0; var minValue = 0;
var maxValue = 0; var maxValue = 0;
var categoriesCount = 0; var categoriesCount = 0;
@ -202,12 +265,15 @@ Aggregation.prototype.format = function(result) {
var firstRow = result.rows[0]; var firstRow = result.rows[0];
count = firstRow.count; count = firstRow.count;
nulls = firstRow.nulls_count; nulls = firstRow.nulls_count;
nans = firstRow.nans_count;
infinities = firstRow.infinities_count;
minValue = firstRow.min_val; minValue = firstRow.min_val;
maxValue = firstRow.max_val; maxValue = firstRow.max_val;
categoriesCount = firstRow.categories_count; categoriesCount = firstRow.categories_count;
result.rows.forEach(function(row) { result.rows.forEach(function(row) {
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); categories.push(_.omit(row, 'count', 'nulls_count', 'min_val',
'max_val', 'categories_count', 'nans_count', 'infinities_count'));
}); });
} }
@ -215,6 +281,8 @@ Aggregation.prototype.format = function(result) {
aggregation: this.aggregation, aggregation: this.aggregation,
count: count, count: count,
nulls: nulls, nulls: nulls,
nans: nans,
infinities: infinities,
min: minValue, min: minValue,
max: maxValue, max: maxValue,
categoriesCount: categoriesCount, categoriesCount: categoriesCount,
@ -253,6 +321,8 @@ Aggregation.prototype.search = function(psql, userQuery, callback) {
var self = this; var self = this;
var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
var _value = this.aggregation !== 'count' && this.aggregationColumn ?
this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)';
// TODO unfiltered will be wrong as filters are already applied at this point // TODO unfiltered will be wrong as filters are already applied at this point
var query = searchQueryTpl({ var query = searchQueryTpl({
@ -265,7 +335,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) {
_searchFiltered: filterCategoriesQueryTpl({ _searchFiltered: filterCategoriesQueryTpl({
_query: this.query, _query: this.query,
_column: this.column, _column: this.column,
_value: 'count(1)', _value: _value,
_userQuery: _userQuery _userQuery: _userQuery
}) })
}); });

View File

@ -1,3 +1,6 @@
var dot = require('dot');
dot.templateSettings.strip = false;
function BaseDataview() {} function BaseDataview() {}
module.exports = BaseDataview; module.exports = BaseDataview;
@ -24,3 +27,42 @@ BaseDataview.prototype.getResult = function(psql, override, callback) {
BaseDataview.prototype.search = function(psql, userQuery, callback) { BaseDataview.prototype.search = function(psql, userQuery, callback) {
return callback(null, this.format({ rows: [] })); return callback(null, this.format({ rows: [] }));
}; };
var FLOAT_OIDS = {
700: true,
701: true,
1700: true
};
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1'
);
BaseDataview.prototype.getColumnType = function (psql, column, query, callback) {
var readOnlyTransaction = true;
var columnTypeQuery = columnTypeQueryTpl({
column: column, query: query
});
psql.query(columnTypeQuery, function(err, result) {
if (err) {
return callback(err);
}
var pgType = result.rows[0].pg_typeof;
callback(null, getPGTypeName(pgType));
}, readOnlyTransaction);
};
function getPGTypeName (pgType) {
return {
float: FLOAT_OIDS.hasOwnProperty(pgType),
date: DATE_OIDS.hasOwnProperty(pgType)
};
}

View File

@ -9,7 +9,17 @@ var formulaQueryTpl = dot.template([
'SELECT', 'SELECT',
' {{=it._operation}}({{=it._column}}) AS result,', ' {{=it._operation}}({{=it._column}}) AS result,',
' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula' ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')); ].join('\n'));
var VALID_OPERATIONS = { var VALID_OPERATIONS = {
@ -31,7 +41,7 @@ var TYPE = 'formula';
} }
} }
*/ */
function Formula(query, options) { function Formula(query, options, queries) {
if (!_.isString(options.operation)) { if (!_.isString(options.operation)) {
throw new Error('Formula expects `operation` in widget options'); throw new Error('Formula expects `operation` in widget options');
} }
@ -47,8 +57,10 @@ function Formula(query, options) {
BaseWidget.apply(this); BaseWidget.apply(this);
this.query = query; this.query = query;
this.queries = queries;
this.column = options.column || '1'; this.column = options.column || '1';
this.operation = options.operation; this.operation = options.operation;
this._isFloatColumn = null;
} }
Formula.prototype = new BaseWidget(); Formula.prototype = new BaseWidget();
@ -57,14 +69,27 @@ Formula.prototype.constructor = Formula;
module.exports = Formula; module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) { Formula.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) { if (!callback) {
callback = override; callback = override;
override = {}; override = {};
} }
var _query = this.query; if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var formulaSql = formulaQueryTpl({ var formulaSql = formulaQueryTpl({
_query: _query, _isFloatColumn: this._isFloatColumn,
_query: this.query,
_operation: this.operation, _operation: this.operation,
_column: this.column _column: this.column
}); });
@ -78,13 +103,17 @@ Formula.prototype.format = function(result) {
var formattedResult = { var formattedResult = {
operation: this.operation, operation: this.operation,
result: 0, result: 0,
nulls: 0 nulls: 0,
nans: 0,
infinities: 0
}; };
if (result.rows.length) { if (result.rows.length) {
formattedResult.operation = this.operation; formattedResult.operation = this.operation;
formattedResult.result = result.rows[0].result; formattedResult.result = result.rows[0].result;
formattedResult.nulls = result.rows[0].nulls_count; formattedResult.nulls = result.rows[0].nulls_count;
formattedResult.nans = result.rows[0].nans_count;
formattedResult.infinities = result.rows[0].infinities_count;
} }
return formattedResult; return formattedResult;

View File

@ -5,11 +5,6 @@ var debug = require('debug')('windshaft:dataview:histogram');
var dot = require('dot'); var dot = require('dot');
dot.templateSettings.strip = false; dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
var dateIntervalQueryTpl = dot.template([ var dateIntervalQueryTpl = dot.template([
'WITH', 'WITH',
'dates AS (', 'dates AS (',
@ -54,12 +49,27 @@ var MAX_INTERVAL_VALUE = 366;
var BIN_MIN_NUMBER = 6; var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48; var BIN_MAX_NUMBER = 48;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' WHERE',
' {{=it._column}} IS NOT NULL',
' {{?it._isFloatColumn}}AND',
' {{=it._column}} != \'infinity\'::float',
' AND',
' {{=it._column}} != \'-infinity\'::float',
' AND',
' {{=it._column}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var basicsQueryTpl = dot.template([ var basicsQueryTpl = dot.template([
'basics AS (', 'basics AS (',
' SELECT', ' SELECT',
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics', ' FROM filtered_source',
')' ')'
].join(' \n')); ].join(' \n'));
@ -68,7 +78,7 @@ var overrideBasicsQueryTpl = dot.template([
' SELECT', ' SELECT',
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics', ' FROM filtered_source',
')' ')'
].join('\n')); ].join('\n'));
@ -79,7 +89,7 @@ var iqrQueryTpl = dot.template([
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile', ' ) AS quartile',
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', ' FROM filtered_source) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3', ' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile', ' GROUP BY quartile',
' ) _cdb_iqr', ' ) _cdb_iqr',
@ -98,7 +108,7 @@ var binsQueryTpl = dot.template([
' )', ' )',
' )', ' )',
' END AS bins_number', ' END AS bins_number',
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', ' FROM basics, iqrange, filtered_source',
' LIMIT 1', ' LIMIT 1',
')' ')'
].join('\n')); ].join('\n'));
@ -118,11 +128,34 @@ var nullsQueryTpl = dot.template([
')' ')'
].join('\n')); ].join('\n'));
var infinitiesQueryTpl = dot.template([
'infinities AS (',
' SELECT',
' count(*) AS infinities_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE',
' {{=it._column}} = \'infinity\'::float',
' OR',
' {{=it._column}} = \'-infinity\'::float',
')'
].join('\n'));
var nansQueryTpl = dot.template([
'nans AS (',
' SELECT',
' count(*) AS nans_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE {{=it._column}} = \'NaN\'::float',
')'
].join('\n'));
var histogramQueryTpl = dot.template([ var histogramQueryTpl = dot.template([
'SELECT', 'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,', ' bins_number,',
' nulls_count,', ' nulls_count,',
' {{?it._isFloatColumn}}infinities_count,',
' nans_count,{{?}}',
' avg_val,', ' avg_val,',
' CASE WHEN min_val = max_val', ' CASE WHEN min_val = max_val',
' THEN 0', ' THEN 0',
@ -132,9 +165,9 @@ var histogramQueryTpl = dot.template([
' max({{=it._column}})::numeric AS max,', ' max({{=it._column}})::numeric AS max,',
' avg({{=it._column}})::numeric AS avg,', ' avg({{=it._column}})::numeric AS avg,',
' count(*) AS freq', ' count(*) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}',
'WHERE {{=it._column}} IS NOT NULL', 'GROUP BY bin, bins_number, bin_width, nulls_count,',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
'ORDER BY bin' 'ORDER BY bin'
].join('\n')); ].join('\n'));
@ -268,54 +301,47 @@ Histogram.prototype.constructor = Histogram;
module.exports = Histogram; module.exports = Histogram;
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
Histogram.prototype.sql = function(psql, override, callback) { Histogram.prototype.sql = function(psql, override, callback) {
// jshint maxcomplexity: 7 var self = this;
if (!callback) { if (!callback) {
callback = override; callback = override;
override = {}; override = {};
} }
var self = this;
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.queries.no_filters
});
if (this._columnType === null) { if (this._columnType === null) {
psql.query(columnTypeQuery, function(err, result) { this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
// assume numeric, will fail later // assume numeric, will fail later
self._columnType = 'numeric'; self._columnType = 'numeric';
if (!err && !!result.rows[0]) { if (!err && !!type) {
var pgType = result.rows[0].pg_typeof; self._columnType = Object.keys(type).find(function (key) {
if (DATE_OIDS.hasOwnProperty(pgType)) { return type[key];
self._columnType = 'date'; });
}
} }
self.sql(psql, override, callback); self.sql(psql, override, callback);
}, true); // use read-only transaction }, true); // use read-only transaction
return null; return null;
} }
this._buildQuery(psql, override, callback);
};
Histogram.prototype._buildQuery = function (psql, override, callback) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.query;
if (this._columnType === 'date') { if (this._columnType === 'date') {
return this._buildDateHistogramQuery(psql, override, callback); return this._buildDateHistogramQuery(psql, override, callback);
} }
if (this._columnType === 'date') { filteredQuery = filteredQueryTpl({
_column = columnCastTpl({ column: this.column}); _isFloatColumn: this._columnType === 'float',
} _query: _query,
var _query = this.query; _column: _column
});
var basicsQuery, binsQuery; if (this._shouldOverride(override)) {
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
debug('overriding with %j', override); debug('overriding with %j', override);
basicsQuery = overrideBasicsQueryTpl({ basicsQuery = overrideBasicsQueryTpl({
_query: _query, _query: _query,
@ -335,7 +361,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
_column: _column _column: _column
}); });
if (override && _.has(override, 'bins')) { if (this._shouldOverrideBins(override)) {
binsQuery = [ binsQuery = [
overrideBinsQueryTpl({ overrideBinsQueryTpl({
_bins: override.bins _bins: override.bins
@ -356,17 +382,34 @@ Histogram.prototype.sql = function(psql, override, callback) {
} }
} }
var histogramSql = [ var cteSql = [
"WITH", filteredQuery,
[
basicsQuery, basicsQuery,
binsQuery, binsQuery,
nullsQueryTpl({ nullsQueryTpl({
_query: _query, _query: _query,
_column: _column _column: _column
}) })
].join(',\n'), ];
if (this._columnType === 'float') {
cteSql.push(
infinitiesQueryTpl({
_query: _query,
_column: _column
}),
nansQueryTpl({
_query: _query,
_column: _column
})
);
}
var histogramSql = [
"WITH",
cteSql.join(',\n'),
histogramQueryTpl({ histogramQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query, _query: _query,
_column: _column _column: _column
}) })
@ -377,6 +420,14 @@ Histogram.prototype.sql = function(psql, override, callback) {
return callback(null, histogramSql); return callback(null, histogramSql);
}; };
Histogram.prototype._shouldOverride = function (override) {
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
};
Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};
Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) {
var _column = this.column; var _column = this.column;
var _query = this.query; var _query = this.query;
@ -492,14 +543,16 @@ Histogram.prototype.format = function(result, override) {
override = override || {}; override = override || {};
var buckets = []; var buckets = [];
var aggregation = getAggregation(override, this.aggregation);
var offset = getOffset(override, this.offset);
var binsCount = getBinsCount(override); var binsCount = getBinsCount(override);
var width = getWidth(override); var width = getWidth(override);
var binsStart = getBinStart(override); var binsStart = getBinStart(override);
var nulls = 0; var nulls = 0;
var infinities = 0;
var nans = 0;
var avg; var avg;
var timestampStart; var timestampStart;
var aggregation;
var offset;
if (result.rows.length) { if (result.rows.length) {
var firstRow = result.rows[0]; var firstRow = result.rows[0];
@ -507,10 +560,27 @@ Histogram.prototype.format = function(result, override) {
width = firstRow.bin_width || width; width = firstRow.bin_width || width;
avg = firstRow.avg_val; avg = firstRow.avg_val;
nulls = firstRow.nulls_count; nulls = firstRow.nulls_count;
binsStart = populateBinStart(override, firstRow);
timestampStart = firstRow.timestamp_start; timestampStart = firstRow.timestamp_start;
infinities = firstRow.infinities_count;
nans = firstRow.nans_count;
binsStart = populateBinStart(override, firstRow);
if (timestampStart) {
aggregation = getAggregation(override, this.aggregation);
offset = getOffset(override, this.offset);
}
buckets = result.rows.map(function(row) { buckets = result.rows.map(function(row) {
return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'timestamp_start'); return _.omit(
row,
'bins_number',
'bin_width',
'nulls_count',
'infinities_count',
'nans_count',
'avg_val',
'timestamp_start'
);
}); });
} }
@ -522,6 +592,8 @@ Histogram.prototype.format = function(result, override) {
bins_count: binsCount, bins_count: binsCount,
bins_start: binsStart, bins_start: binsStart,
nulls: nulls, nulls: nulls,
infinities: infinities,
nans: nans,
avg: avg, avg: avg,
bins: buckets bins: buckets
}; };

View File

@ -1,14 +1,36 @@
var BaseOverviewsDataview = require('./base'); var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../aggregation'); var BaseDataview = require('../aggregation');
var debug = require('debug')('windshaft:widget:aggregation:overview');
var dot = require('dot'); var dot = require('dot');
dot.templateSettings.strip = false; dot.templateSettings.strip = false;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var summaryQueryTpl = dot.template([ var summaryQueryTpl = dot.template([
'summary AS (', 'summary AS (',
' SELECT', ' SELECT',
' sum(_feature_count) AS count,', ' sum(_feature_count) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' {{?it._isFloatColumn}},sum(',
' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1',
' ELSE 0',
' END',
' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls', ' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')' ')'
].join('\n')); ].join('\n'));
@ -17,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
'categories AS(', 'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all', ' FROM filtered_source',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}', ' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC', ' ORDER BY 2 DESC',
@ -36,40 +58,46 @@ var categoriesSummaryCountQueryTpl = dot.template([
' SELECT count(1) AS categories_count', ' SELECT count(1) AS categories_count',
' FROM (', ' FROM (',
' SELECT {{=it._column}} AS category', ' SELECT {{=it._column}} AS category',
' FROM ({{=it._query}}) _cdb_categories', ' FROM filtered_source',
' GROUP BY {{=it._column}}', ' GROUP BY {{=it._column}}',
' ) _cdb_categories_count', ' ) _cdb_categories_count',
')' ')'
].join('\n')); ].join('\n'));
var rankedAggregationQueryTpl = dot.template([ var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}', ' WHERE rank < {{=it._limit}}',
'UNION ALL', 'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}', ' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count' 'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n')); ].join('\n'));
var aggregationQueryTpl = dot.template([ var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count', ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', 'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC' 'ORDER BY value DESC'
].join('\n')); ].join('\n'));
var CATEGORIES_LIMIT = 6; var CATEGORIES_LIMIT = 6;
function Aggregation(query, options, queryRewriter, queryRewriteData, params) { function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.query = query; this.query = query;
this.queries = queries;
this.column = options.column; this.column = options.column;
this.aggregation = options.aggregation; this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn; this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
} }
Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
@ -78,27 +106,49 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation; module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) { Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) { if (!callback) {
callback = override; callback = override;
override = {}; override = {};
} }
var _query = this.rewrittenQuery(this.query); var _query = this.rewrittenQuery(this.query);
var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var aggregationSql; var aggregationSql;
if (!!override.ownFilter) { if (!!override.ownFilter) {
aggregationSql = [ aggregationSql = [
"WITH", "WITH",
[ [
summaryQueryTpl({ filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column _column: this.column,
_aggregationColumn: _aggregationColumn
}),
summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationColumn: _aggregationColumn
}), }),
rankedCategoriesQueryTpl({ rankedCategoriesQueryTpl({
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregation: this.getAggregationSql(), _aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null _aggregationColumn: _aggregationColumn
}), }),
categoriesSummaryMinMaxQueryTpl({ categoriesSummaryMinMaxQueryTpl({
_query: _query, _query: _query,
@ -110,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
}) })
].join(',\n'), ].join(',\n'),
aggregationQueryTpl({ aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregation: this.getAggregationSql(), _aggregation: this.getAggregationSql(),
@ -120,15 +171,23 @@ Aggregation.prototype.sql = function(psql, override, callback) {
aggregationSql = [ aggregationSql = [
"WITH", "WITH",
[ [
summaryQueryTpl({ filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column _column: this.column,
_aggregationColumn: _aggregationColumn
}),
summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationColumn: _aggregationColumn
}), }),
rankedCategoriesQueryTpl({ rankedCategoriesQueryTpl({
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_aggregation: this.getAggregationSql(), _aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null _aggregationColumn: _aggregationColumn
}), }),
categoriesSummaryMinMaxQueryTpl({ categoriesSummaryMinMaxQueryTpl({
_query: _query, _query: _query,
@ -140,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
}) })
].join(',\n'), ].join(',\n'),
rankedAggregationQueryTpl({ rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query, _query: _query,
_column: this.column, _column: this.column,
_limit: CATEGORIES_LIMIT _limit: CATEGORIES_LIMIT
@ -147,6 +207,8 @@ Aggregation.prototype.sql = function(psql, override, callback) {
].join('\n'); ].join('\n');
} }
debug(aggregationSql);
return callback(null, aggregationSql); return callback(null, aggregationSql);
}; };

View File

@ -1,14 +1,15 @@
var _ = require('underscore'); var _ = require('underscore');
var BaseDataview = require('../base'); var BaseDataview = require('../base');
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) {
this.BaseDataview = BaseDataview; this.BaseDataview = BaseDataview;
this.query = query; this.query = query;
this.queryOptions = queryOptions; this.queryOptions = queryOptions;
this.queryRewriter = queryRewriter; this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData; this.queryRewriteData = queryRewriteData;
this.options = options; this.options = options;
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions); this.queries = queries;
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries);
} }
module.exports = BaseOverviewsDataview; module.exports = BaseOverviewsDataview;

View File

@ -1,5 +1,6 @@
var BaseOverviewsDataview = require('./base'); var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../formula'); var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overview');
var dot = require('dot'); var dot = require('dot');
dot.templateSettings.strip = false; dot.templateSettings.strip = false;
@ -9,26 +10,52 @@ var formulaQueryTpls = {
'SELECT', 'SELECT',
'sum(_feature_count) AS result,', 'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula' 'FROM ({{=it._query}}) _cdb_formula'
].join('\n')), ].join('\n')),
'sum': dot.template([ 'sum': dot.template([
'SELECT', 'SELECT',
'sum({{=it._column}}*_feature_count) AS result,', 'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula' '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')), ].join('\n')),
'avg': dot.template([ 'avg': dot.template([
'SELECT', 'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula' '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')), ].join('\n')),
}; };
function Formula(query, options, queryRewriter, queryRewriteData, params) { function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.column = options.column || '1'; this.column = options.column || '1';
this.operation = options.operation; this.operation = options.operation;
this._isFloatColumn = null;
this.queries = queries;
} }
Formula.prototype = Object.create(BaseOverviewsDataview.prototype); Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
@ -37,20 +64,37 @@ Formula.prototype.constructor = Formula;
module.exports = Formula; module.exports = Formula;
Formula.prototype.sql = function (psql, override, callback) { Formula.prototype.sql = function (psql, override, callback) {
var self = this;
var formulaQueryTpl = formulaQueryTpls[this.operation]; var formulaQueryTpl = formulaQueryTpls[this.operation];
if (formulaQueryTpl) { if (formulaQueryTpl) {
// supported formula for use with overviews // supported formula for use with overviews
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var formulaSql = formulaQueryTpl({ var formulaSql = formulaQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: this.rewrittenQuery(this.query), _query: this.rewrittenQuery(this.query),
_operation: this.operation, _operation: this.operation,
_column: this.column _column: this.column
}); });
callback = callback || override; callback = callback || override;
debug(formulaSql);
return callback(null, formulaSql); return callback(null, formulaSql);
} }
// default behaviour // default behaviour
return this.defaultSql(psql, override, callback); return this.defaultSql(psql, override, callback);
}; };

View File

@ -1,23 +1,35 @@
var _ = require('underscore'); var _ = require('underscore');
var BaseOverviewsDataview = require('./base'); var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../histogram'); var BaseDataview = require('../histogram');
var debug = require('debug')('windshaft:dataview:histogram:overview');
var dot = require('dot'); var dot = require('dot');
dot.templateSettings.strip = false; dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var BIN_MIN_NUMBER = 6; var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48; var BIN_MAX_NUMBER = 48;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' WHERE',
' {{=it._column}} IS NOT NULL',
' {{?it._isFloatColumn}}AND',
' {{=it._column}} != \'infinity\'::float',
' AND',
' {{=it._column}} != \'-infinity\'::float',
' AND',
' {{=it._column}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var basicsQueryTpl = dot.template([ var basicsQueryTpl = dot.template([
'basics AS (', 'basics AS (',
' SELECT', ' SELECT',
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics', ' FROM filtered_source',
')' ')'
].join(' \n')); ].join(' \n'));
@ -26,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([
' SELECT', ' SELECT',
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics', ' FROM filtered_source',
')' ')'
].join('\n')); ].join('\n'));
@ -37,7 +49,7 @@ var iqrQueryTpl = dot.template([
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile', ' ) AS quartile',
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', ' FROM filtered_source) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3', ' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile', ' GROUP BY quartile',
' ) _cdb_iqr', ' ) _cdb_iqr',
@ -56,7 +68,7 @@ var binsQueryTpl = dot.template([
' )', ' )',
' )', ' )',
' END AS bins_number', ' END AS bins_number',
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', ' FROM basics, iqrange, filtered_source',
' LIMIT 1', ' LIMIT 1',
')' ')'
].join('\n')); ].join('\n'));
@ -76,11 +88,34 @@ var nullsQueryTpl = dot.template([
')' ')'
].join('\n')); ].join('\n'));
var infinitiesQueryTpl = dot.template([
'infinities AS (',
' SELECT',
' count(*) AS infinities_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE',
' {{=it._column}} = \'infinity\'::float',
' OR',
' {{=it._column}} = \'-infinity\'::float',
')'
].join('\n'));
var nansQueryTpl = dot.template([
'nans AS (',
' SELECT',
' count(*) AS nans_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE {{=it._column}} = \'NaN\'::float',
')'
].join('\n'));
var histogramQueryTpl = dot.template([ var histogramQueryTpl = dot.template([
'SELECT', 'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,', ' bins_number,',
' nulls_count,', ' nulls_count,',
' {{?it._isFloatColumn}}infinities_count,',
' nans_count,{{?}}',
' avg_val,', ' avg_val,',
' CASE WHEN min_val = max_val', ' CASE WHEN min_val = max_val',
' THEN 0', ' THEN 0',
@ -90,14 +125,14 @@ var histogramQueryTpl = dot.template([
' max({{=it._column}})::numeric AS max,', ' max({{=it._column}})::numeric AS max,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
' sum(_feature_count) AS freq', ' sum(_feature_count) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}',
'WHERE {{=it._column}} IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
'ORDER BY bin' 'ORDER BY bin'
].join('\n')); ].join('\n'));
function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) { function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.query = query; this.query = query;
this.queries = queries; this.queries = queries;
@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram;
module.exports = Histogram; module.exports = Histogram;
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
Histogram.prototype.sql = function(psql, override, callback) { Histogram.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) { if (!callback) {
callback = override; callback = override;
override = {}; override = {};
} }
var self = this;
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.rewrittenQuery(this.queries.no_filters)
});
if (this._columnType === null) { if (this._columnType === null) {
psql.query(columnTypeQuery, function(err, result) { this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
// assume numeric, will fail later // assume numeric, will fail later
self._columnType = 'numeric'; self._columnType = 'numeric';
if (!err && !!result.rows[0]) { if (!err && !!type) {
var pgType = result.rows[0].pg_typeof; self._columnType = Object.keys(type).find(function (key) {
if (DATE_OIDS.hasOwnProperty(pgType)) { return type[key];
self._columnType = 'date'; });
}
} }
self.sql(psql, override, callback); self.sql(psql, override, callback);
}, true); // use read-only transaction }, true); // use read-only transaction
@ -154,11 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) {
return this.defaultSql(psql, override, callback); return this.defaultSql(psql, override, callback);
} }
var histogramSql = this._buildQuery(override);
return callback(null, histogramSql);
};
Histogram.prototype._buildQuery = function (override) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.rewrittenQuery(this.query); var _query = this.rewrittenQuery(this.query);
var basicsQuery, binsQuery; filteredQuery = filteredQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
});
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { if (this._shouldOverride(override)) {
debug('overriding with %j', override);
basicsQuery = overrideBasicsQueryTpl({ basicsQuery = overrideBasicsQueryTpl({
_query: _query, _query: _query,
_column: _column, _column: _column,
@ -177,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
_column: _column _column: _column
}); });
if (override && _.has(override, 'bins')) { if (this._shouldOverrideBins(override)) {
binsQuery = [ binsQuery = [
overrideBinsQueryTpl({ overrideBinsQueryTpl({
_bins: override.bins _bins: override.bins
@ -198,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) {
} }
} }
var cteSql = [
var histogramSql = [ filteredQuery,
"WITH",
[
basicsQuery, basicsQuery,
binsQuery, binsQuery,
nullsQueryTpl({ nullsQueryTpl({
_query: _query, _query: _query,
_column: _column _column: _column
}) })
].join(',\n'), ];
if (this._columnType === 'float') {
cteSql.push(
infinitiesQueryTpl({
_query: _query,
_column: _column
}),
nansQueryTpl({
_query: _query,
_column: _column
})
);
}
var histogramSql = [
"WITH",
cteSql.join(',\n'),
histogramQueryTpl({ histogramQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query, _query: _query,
_column: _column _column: _column
}) })
].join('\n'); ].join('\n');
return callback(null, histogramSql); debug(histogramSql);
return histogramSql;
}; };
Histogram.prototype._shouldOverride = function (override) {
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
};
Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};

View File

@ -1,8 +1,8 @@
var BaseOverviewsDataview = require('./base'); var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../list'); var BaseDataview = require('../list');
function List(query, options, queryRewriter, queryRewriteData, params) { function List(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
} }
List.prototype = Object.create(BaseOverviewsDataview.prototype); List.prototype = Object.create(BaseOverviewsDataview.prototype);

View File

@ -1,7 +1,7 @@
{ {
"private": true, "private": true,
"name": "windshaft-cartodb", "name": "windshaft-cartodb",
"version": "3.9.3", "version": "3.9.9",
"description": "A map tile server for CartoDB", "description": "A map tile server for CartoDB",
"keywords": [ "keywords": [
"cartodb" "cartodb"
@ -16,11 +16,12 @@
"contributors": [ "contributors": [
"Simon Tokumine <simon@vizzuality.com>", "Simon Tokumine <simon@vizzuality.com>",
"Javi Santana <jsantana@vizzuality.com>", "Javi Santana <jsantana@vizzuality.com>",
"Sandro Santilli <strk@vizzuality.com>" "Sandro Santilli <strk@vizzuality.com>",
"Carlos Matallín <matallo@carto.com>"
], ],
"dependencies": { "dependencies": {
"body-parser": "~1.14.0", "body-parser": "~1.14.0",
"camshaft": "0.55.5", "camshaft": "0.55.6",
"cartodb-psql": "0.8.0", "cartodb-psql": "0.8.0",
"cartodb-query-tables": "0.2.0", "cartodb-query-tables": "0.2.0",
"cartodb-redis": "0.13.2", "cartodb-redis": "0.13.2",
@ -37,9 +38,9 @@
"request": "~2.79.0", "request": "~2.79.0",
"step": "~0.0.6", "step": "~0.0.6",
"step-profiler": "~0.3.0", "step-profiler": "~0.3.0",
"turbo-carto": "0.19.1", "turbo-carto": "0.19.2",
"underscore": "~1.6.0", "underscore": "~1.6.0",
"windshaft": "3.2.1", "windshaft": "3.2.2",
"yargs": "~5.0.0" "yargs": "~5.0.0"
}, },
"devDependencies": { "devDependencies": {

View File

@ -145,4 +145,182 @@ describe('aggregations happy cases', function() {
}); });
}); });
}); });
var widgetSearchExpects = {
'count': [ { category: 'other_a', value: 3 } ],
'sum': [ { category: 'other_a', value: 6 } ],
'avg': [ { category: 'other_a', value: 2 } ],
'max': [ { category: 'other_a', value: 3 } ],
'min': [ { category: 'other_a', value: 1 } ]
};
Object.keys(operations_and_values).forEach(function (operation) {
var description = 'should search OTHER category using "' + operation + '"';
it(description, function (done) {
this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) {
assert.ifError(err);
assert.ok(searchResult);
assert.equal(searchResult.type, 'aggregation');
assert.equal(searchResult.categories.length, 1);
assert.deepEqual(
searchResult.categories,
widgetSearchExpects[operation]
);
done();
});
});
});
});
describe('aggregation-dataview: special float values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
val_aggregation: {
source: {
id: 'a0'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'avg',
aggregationColumn: 'val'
}
},
sum_aggregation_numeric: {
source: {
id: 'a1'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'sum',
aggregationColumn: 'val'
}
}
},
[
{
"id": "a0",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS val,',
' CASE',
' WHEN x % 2 = 0 THEN \'category_1\'',
' ELSE \'category_2\'',
' END AS cat',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}, {
"id": "a1",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 3 = 0 THEN \'NaN\'::numeric',
' WHEN x % 3 = 1 THEN x',
' ELSE x',
' END AS val,',
' CASE',
' WHEN x % 2 = 0 THEN \'category_1\'',
' ELSE \'category_2\'',
' END AS cat',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}
]
);
// Source a0
// -----------------------------------------------
// the_geom_webmercator | val | cat
// ----------------------+-----------+------------
// | -Infinity | category_2
// | NaN | category_1
// | 3 | category_2
// | Infinity | category_1
// | -Infinity | category_2
// | NaN | category_1
// | 7 | category_2
// | Infinity | category_1
// | -Infinity | category_2
// | NaN | category_1
// | 11 | category_2
// | " | "
var filters = [{ own_filter: 0 }, {}];
filters.forEach(function (filter) {
it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) {
assert.ifError(err);
assert.ok(dataview.infinities === (250 + 250));
assert.ok(dataview.nans === 250);
assert.ok(dataview.categories.length === 1);
dataview.categories.forEach(function (category) {
assert.ok(category.category === 'category_2');
assert.ok(category.value === 501);
});
done();
});
});
it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) {
assert.ifError(err);
assert.ok(dataview.nans === 333);
assert.ok(dataview.categories.length === 2);
dataview.categories.forEach(function (category) {
assert.ok(category.value !== null);
});
done();
});
});
});
}); });

View File

@ -0,0 +1,80 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
describe('formula-dataview: special float values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
val_formula: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: 'val',
operation: 'avg'
}
}
},
[
{
"id": "a0",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS val',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}
]
);
it('should filter infinities out and count them in the summary', function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('val_formula', {}, function(err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.result, 501);
assert.ok(dataview.infinities === (250 + 250));
assert.ok(dataview.nans === 250);
done();
});
});
});

View File

@ -13,6 +13,15 @@ function createMapConfig(layers, dataviews, analysis) {
}; };
} }
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
describe('histogram-dataview', function() { describe('histogram-dataview', function() {
afterEach(function(done) { afterEach(function(done) {
@ -90,7 +99,6 @@ describe('histogram-dataview', function() {
this.testClient = new TestClient(mapConfig, 1234); this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('pop_max_histogram', params, function(err, res) { this.testClient.getDataview('pop_max_histogram', params, function(err, res) {
assert.ok(!err, err); assert.ok(!err, err);
assert.ok(res.errors); assert.ok(res.errors);
assert.equal(res.errors.length, 1); assert.equal(res.errors.length, 1);
assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/));
@ -689,3 +697,70 @@ describe('histogram-dataview for date column type', function() {
}); });
}); });
}); });
describe('histogram-dataview: special float valuer', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
val_histogram: {
source: {
id: 'a0'
},
type: 'histogram',
options: {
column: 'val'
}
}
},
[
{
"id": "a0",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS val',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}
]
);
it('should filter infinities out and count them in the summary', function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('val_histogram', {}, function(err, dataview) {
assert.ok(!err, err);
assert.ok(dataview.infinities === (250 + 250));
assert.ok(dataview.nans === 250);
done();
});
});
});

View File

@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() {
params: { params: {
query: 'select * from test_table_overviews' query: 'select * from test_table_overviews'
} }
},
{
id: 'data-source-special-float-values',
type: 'source',
params: {
query: 'select * from test_special_float_values_table_overviews'
}
} }
], ],
dataviews: { dataviews: {
@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() {
aggregationColumn: 'name', aggregationColumn: 'name',
} }
}, },
test_categories_special_values: {
type: 'aggregation',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'name',
aggregation: 'sum',
aggregationColumn: 'value',
}
},
test_histogram: { test_histogram: {
type: 'histogram', type: 'histogram',
source: {id: 'data-source'}, source: {id: 'data-source'},
@ -160,6 +178,16 @@ describe('dataviews using tables with overviews', function() {
bins: 2 bins: 2
} }
}, },
test_histogram_special_values: {
type: 'histogram',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'value',
bins: 2
}
},
test_avg: { test_avg: {
type: 'formula', type: 'formula',
source: {id: 'data-source'}, source: {id: 'data-source'},
@ -168,6 +196,16 @@ describe('dataviews using tables with overviews', function() {
operation: 'avg' operation: 'avg'
} }
}, },
test_formula_sum_special_values: {
type: 'formula',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'value',
operation: 'sum'
}
},
test_count: { test_count: {
type: 'formula', type: 'formula',
source: {id: 'data-source'}, source: {id: 'data-source'},
@ -202,6 +240,17 @@ describe('dataviews using tables with overviews', function() {
cartocss_version: '2.3.0', cartocss_version: '2.3.0',
source: { id: 'data-source' } source: { id: 'data-source' }
} }
},
{
type: 'mapnik',
options: {
sql: 'select * from test_special_float_values_table_overviews',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
source: {
id: 'data-source-special-float-values'
}
}
} }
] ]
}; };
@ -212,7 +261,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"sum",
"result":15,
"infinities": 0,
"nans": 0,
"nulls":0,
"type":"formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -224,7 +280,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"avg",
"result":3,
"nulls":0,
"type":"formula",
"infinities": 0,
"nans": 0
});
testClient.drain(done); testClient.drain(done);
}); });
@ -236,7 +299,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"count",
"result":5,
"nulls":0,
"type":"formula",
"infinities": 0,
"nans": 0
});
testClient.drain(done); testClient.drain(done);
}); });
@ -248,7 +318,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation": "max",
"result": 5,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -260,7 +337,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation": "min",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -275,7 +359,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"sum",
"result":15,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -372,7 +463,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"sum",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done); testClient.drain(done);
}); });
}); });
@ -383,7 +481,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"avg",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -395,7 +500,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"count",
"result":1,
"infinities": 0,
"nans": 0,
"nulls":0,
"type":"formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -407,7 +519,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation": "max",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -419,7 +538,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation": "min",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done); testClient.drain(done);
}); });
@ -437,7 +563,14 @@ describe('dataviews using tables with overviews', function() {
if (err) { if (err) {
return done(err); return done(err);
} }
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); assert.deepEqual(formula_result, {
"operation":"sum",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done); testClient.drain(done);
}); });
}); });
@ -445,5 +578,69 @@ describe('dataviews using tables with overviews', function() {
}); });
describe('aggregation special float values', function () {
var params = {};
it("should expose an aggregation dataview filtering special float values out", function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_categories_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
aggregation: 'sum',
count: 5,
nulls: 0,
nans: 1,
infinities: 1,
min: 6,
max: 6,
categoriesCount: 1,
categories: [ { category: 'Hawai', value: 6, agg: false } ],
type: 'aggregation'
});
testClient.drain(done);
});
});
it('should expose a histogram dataview filtering special float values out', function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_histogram_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
bin_width: 0,
bins_count: 1,
bins_start: 3,
nulls: 0,
infinities: 1,
nans: 1,
avg: 3,
bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ],
type: 'histogram'
});
testClient.drain(done);
});
});
it('should expose a formula (sum) dataview filtering special float values out', function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
operation: 'sum',
result: 6,
nulls: 0,
nans: 1,
infinities: 1,
type: 'formula'
});
testClient.drain(done);
});
});
});
}); });
}); });

53
test/acceptance/mvt.js Normal file
View File

@ -0,0 +1,53 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
function createMapConfig (sql = TestClient.SQL.ONE_POINT) {
return {
version: '1.6.0',
layers: [{
type: "cartodb",
options: {
sql: sql,
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
}]
};
}
describe('mvt', function () {
const testCases = [
{
desc: 'should get empty mvt with code 204 (no content)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
status: 204,
mapConfig: createMapConfig(TestClient.SQL.EMPTY)
},
{
desc: 'should get mvt tile with code 200 (ok)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
status: 200,
mapConfig: createMapConfig()
}
];
testCases.forEach(function (test) {
it(test.desc, done => {
const testClient = new TestClient(test.mapConfig, 1234);
const { z, x, y } = test.coords;
const { format, status } = test;
testClient.getTile(z, x, y, { format, status }, (err, res) => {
assert.ifError(err);
assert.equal(res.statusCode, test.status);
testClient.drain(done);
});
});
});
});

View File

@ -322,6 +322,25 @@ describe('widgets', function() {
}); });
}); });
}); });
[adm0name].forEach(function(userQuery) {
it('should search with sum aggregation: ' + userQuery, function(done) {
this.testClient = new TestClient(aggregationSumMapConfig);
this.testClient.widgetSearch('adm0name', userQuery, function (err, res, searchResult) {
assert.ok(!err, err);
assert.ok(searchResult);
assert.equal(searchResult.type, 'aggregation');
assert.equal(searchResult.categories.length, 1);
assert.deepEqual(
searchResult.categories,
[{ category:"Argentina", value:28015640 }]
);
done();
});
});
});
}); });
}); });

View File

@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES
INSERT INTO _vovw_1_test_table_overviews VALUES INSERT INTO _vovw_1_test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5); ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5);
-- table with overviews whit special float values
CREATE TABLE test_special_float_values_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id;
SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true);
ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass);
INSERT INTO test_special_float_values_table_overviews VALUES
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1),
(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1),
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1),
(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1);
ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id);
CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom);
CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator);
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
CREATE TABLE _vovw_1_test_special_float_values_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER;
INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2),
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2);
-- analysis tables ----------------------------------------------- -- analysis tables -----------------------------------------------

View File

@ -75,6 +75,11 @@ module.exports.CARTOCSS = {
].join('\n') ].join('\n')
}; };
module.exports.SQL = {
EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator',
ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator'
}
TestClient.prototype.getWidget = function(widgetName, params, callback) { TestClient.prototype.getWidget = function(widgetName, params, callback) {
var self = this; var self = this;
@ -525,7 +530,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}; };
var expectedResponse = { var expectedResponse = {
status: 200, status: params.status || 200,
headers: { headers: {
'Content-Type': 'application/json; charset=utf-8' 'Content-Type': 'application/json; charset=utf-8'
} }
@ -542,7 +547,12 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
if (isMvt) { if (isMvt) {
request.encoding = 'binary'; request.encoding = 'binary';
if (expectedResponse.status === 200) {
expectedResponse.headers['Content-Type'] = 'application/x-protobuf'; expectedResponse.headers['Content-Type'] = 'application/x-protobuf';
} else if (expectedResponse.status === 204) {
expectedResponse.headers['Content-Type'] = undefined;
}
} }
var isGeojson = format.match(/geojson$/); var isGeojson = format.match(/geojson$/);
@ -561,16 +571,17 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
assert.response(server, request, expectedResponse, function(res, err) { assert.response(server, request, expectedResponse, function(res, err) {
assert.ifError(err); assert.ifError(err);
var obj; var obj;
if (isPng) { if (isPng) {
obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
} }
else if (isMvt) { else if (isMvt) {
if (res.body) {
obj = new mapnik.VectorTile(z, x, y); obj = new mapnik.VectorTile(z, x, y);
obj.setDataSync(new Buffer(res.body, 'binary')); obj.setDataSync(new Buffer(res.body, 'binary'));
} }
}
else { else {
obj = JSON.parse(res.body); obj = JSON.parse(res.body);
} }

View File

@ -2,7 +2,7 @@
# yarn lockfile v1 # yarn lockfile v1
abaculus@cartodb/abaculus#2.0.3-cdb1: "abaculus@github:cartodb/abaculus#2.0.3-cdb1":
version "2.0.3-cdb1" version "2.0.3-cdb1"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf"
dependencies: dependencies:
@ -53,8 +53,8 @@ ap@~0.2.0:
resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110"
aproba@^1.0.3: aproba@^1.0.3:
version "1.1.1" version "1.1.2"
resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1"
are-we-there-yet@~1.1.2: are-we-there-yet@~1.1.2:
version "1.1.4" version "1.1.4"
@ -161,10 +161,6 @@ browser-stdout@1.3.0:
version "1.3.0" version "1.3.0"
resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f"
buffer-shims@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51"
buffer-writer@1.0.1: buffer-writer@1.0.1:
version "1.0.1" version "1.0.1"
resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08"
@ -198,9 +194,9 @@ camelcase@^3.0.0:
version "3.0.0" version "3.0.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
camshaft@0.55.5: camshaft@0.55.6:
version "0.55.5" version "0.55.6"
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.5.tgz#6831f74022b06e12ddab8e00953c7cc859598ac4" resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.6.tgz#11af28051c3b911fb023ae1cafb165bbd040f174"
dependencies: dependencies:
async "^1.5.2" async "^1.5.2"
bunyan "1.8.1" bunyan "1.8.1"
@ -209,7 +205,7 @@ camshaft@0.55.5:
dot "^1.0.3" dot "^1.0.3"
request "^2.69.0" request "^2.69.0"
canvas@cartodb/node-canvas#1.6.2-cdb2: "canvas@github:cartodb/node-canvas#1.6.2-cdb2":
version "1.6.2-cdb2" version "1.6.2-cdb2"
resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
dependencies: dependencies:
@ -235,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1:
optimist "~0.6.0" optimist "~0.6.0"
underscore "~1.6.0" underscore "~1.6.0"
carto@cartodb/carto#0.15.1-cdb3: "carto@github:cartodb/carto#0.15.1-cdb3":
version "0.15.1-cdb3" version "0.15.1-cdb3"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
dependencies: dependencies:
@ -899,7 +895,7 @@ inflight@^1.0.4:
once "^1.3.0" once "^1.3.0"
wrappy "1" wrappy "1"
inherits@2, inherits@~2.0.0, inherits@~2.0.1: inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
version "2.0.3" version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
@ -1348,8 +1344,8 @@ nock@~2.11.0:
propagate "0.3.x" propagate "0.3.x"
node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31: node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31:
version "0.6.34" version "0.6.36"
resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786"
dependencies: dependencies:
mkdirp "^0.5.1" mkdirp "^0.5.1"
nopt "^4.0.1" nopt "^4.0.1"
@ -1388,8 +1384,8 @@ normalize-package-data@^2.3.2:
validate-npm-package-license "^3.0.1" validate-npm-package-license "^3.0.1"
npmlog@^4.0.2: npmlog@^4.0.2:
version "4.1.0" version "4.1.2"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.0.tgz#dc59bee85f64f00ed424efb2af0783df25d1c0b5" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b"
dependencies: dependencies:
are-we-there-yet "~1.1.2" are-we-there-yet "~1.1.2"
console-control-strings "~1.1.0" console-control-strings "~1.1.0"
@ -1708,15 +1704,15 @@ readable-stream@1.1, readable-stream@~1.1.9:
string_decoder "~0.10.x" string_decoder "~0.10.x"
readable-stream@^2.0.6, readable-stream@^2.1.4: readable-stream@^2.0.6, readable-stream@^2.1.4:
version "2.2.9" version "2.3.3"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c"
dependencies: dependencies:
buffer-shims "~1.0.0"
core-util-is "~1.0.0" core-util-is "~1.0.0"
inherits "~2.0.1" inherits "~2.0.3"
isarray "~1.0.0" isarray "~1.0.0"
process-nextick-args "~1.0.6" process-nextick-args "~1.0.6"
string_decoder "~1.0.0" safe-buffer "~5.1.1"
string_decoder "~1.0.3"
util-deprecate "~1.0.1" util-deprecate "~1.0.1"
readable-stream@~1.0.2: readable-stream@~1.0.2:
@ -1745,7 +1741,7 @@ repeat-string@^1.5.2:
version "1.6.1" version "1.6.1"
resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
request@2.x, request@^2.55.0, request@~2.79.0: request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0:
version "2.79.0" version "2.79.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
dependencies: dependencies:
@ -1770,7 +1766,7 @@ request@2.x, request@^2.55.0, request@~2.79.0:
tunnel-agent "~0.4.1" tunnel-agent "~0.4.1"
uuid "^3.0.0" uuid "^3.0.0"
request@^2.69.0, request@^2.81.0: request@^2.81.0:
version "2.81.0" version "2.81.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0"
dependencies: dependencies:
@ -1827,9 +1823,9 @@ rimraf@~2.4.0:
dependencies: dependencies:
glob "^6.0.1" glob "^6.0.1"
safe-buffer@^5.0.1: safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
version "5.0.1" version "5.1.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
safe-json-stringify@~1: safe-json-stringify@~1:
version "1.0.4" version "1.0.4"
@ -2029,11 +2025,11 @@ string_decoder@~0.10.x:
version "0.10.31" version "0.10.31"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
string_decoder@~1.0.0: string_decoder@~1.0.3:
version "1.0.0" version "1.0.3"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab"
dependencies: dependencies:
buffer-shims "~1.0.0" safe-buffer "~5.1.0"
stringstream@~0.0.4: stringstream@~0.0.4:
version "0.0.5" version "0.0.5"
@ -2107,15 +2103,15 @@ through@2:
version "2.3.8" version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: "tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3":
version "2.3.1-cdb2" version "2.3.1-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23"
dependencies: dependencies:
mapnik "~3.5.0" mapnik "~3.5.0"
mapnik-pool "~0.1.3" mapnik-pool "~0.1.3"
sphericalmercator "1.0.x" sphericalmercator "1.0.x"
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: "tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2":
version "0.6.18-cdb2" version "0.6.18-cdb2"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858"
dependencies: dependencies:
@ -2156,9 +2152,9 @@ tunnel-agent@~0.4.1:
version "0.4.3" version "0.4.3"
resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb"
turbo-carto@0.19.1: turbo-carto@0.19.2:
version "0.19.1" version "0.19.2"
resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.1.tgz#c32af073936a4e8f197dfea918e7441c949d7865" resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd"
dependencies: dependencies:
cartocolor "4.0.0" cartocolor "4.0.0"
colorbrewer "1.0.0" colorbrewer "1.0.0"
@ -2278,9 +2274,9 @@ window-size@^0.2.0:
version "0.2.0" version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@3.2.1: windshaft@3.2.2:
version "3.2.1" version "3.2.2"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.1.tgz#50a3afa6562315dd9e65e411660970e118f36c19" resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.2.tgz#7afb9d8fd8bba1bf02d39c06e8bbe5a451aad953"
dependencies: dependencies:
abaculus cartodb/abaculus#2.0.3-cdb1 abaculus cartodb/abaculus#2.0.3-cdb1
canvas cartodb/node-canvas#1.6.2-cdb2 canvas cartodb/node-canvas#1.6.2-cdb2
@ -2297,7 +2293,7 @@ windshaft@3.2.1:
sphericalmercator "1.0.4" sphericalmercator "1.0.4"
step "~0.0.6" step "~0.0.6"
tilelive "5.12.2" tilelive "5.12.2"
tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb2 tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb3
tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2 tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2
torque.js "~2.11.0" torque.js "~2.11.0"
underscore "~1.6.0" underscore "~1.6.0"