cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into stats-middleware

Browse Source
This commit is contained in:
Daniel García Aubert 2017-10-10 16:56:35 +02:00
commit 02304dc450
25 changed files with 1588 additions and 2009 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.jshintignore
View File

@ -1,4 +1,3 @@
test/results/
test/monkey/
test/benchmark.js
test/support/

2
INSTALL.md
View File

@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are
- Core
- Node.js >=6.9.x
- yarn >=0.21.3
- yarn >=0.27.5 <1.0.0
- PostgreSQL >8.3.x, PostGIS >1.5.x
- Redis >2.4.0 (http://www.redis.io)
- Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).

32
NEWS.md
View File

@ -1,11 +1,39 @@
# Changelog
## 3.12.11
## 4.0.1
Released 2017-mm-dd
- Split and move `req2params` method to multiple middlewares.
- Use express error handler middleware to respond in case of something went wrong.
- Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params.
- Move `LZMA` decompression to its own middleware.
## 4.0.0
Released 2017-10-04
Backward incompatible changes:
- Removes `list` dataview type.
Announcements:
- Upgrades body-parser to 1.18.2.
- Upgrades express to 4.16.0.
- Upgrades debug to 3.1.0.
- Upgrades request to 2.83.0.
- Upgrades turbo-carto to [0.20.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.1)
- Upgrades cartodb-psql to [0.10.2](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.2).
- Upgrades camshaft to [0.59.2](https://github.com/CartoDB/camshaft/releases/tag/0.59.2).
- Upgrades windshaft to [3.3.3](https://github.com/CartoDB/windshaft/releases/tag/3.3.3).
- Upgrades yarn minimum version requirement to v0.27.5
## 3.13.0
Released 2017-10-02
- Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables.
Bugfixes:
- Bounding box parameter ignored in static named maps #735.
- camhaft 0.59.1 fixes duplicate columns in aggregate-intersection analysis
## 3.12.10
Released 2017-09-18

41
docs/MapConfig-Dataviews-extension.md
View File

@ -8,50 +8,13 @@ This specification describes an extension for
This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`.
It makes possible to get tabular data from analysis nodes: lists, aggregated lists, aggregations, and histograms.
It makes possible to get tabular data from analysis nodes: aggregated lists, aggregations, and histograms.
## 2.1. Dataview types
### List
A list is a simple result set per row where is possible to retrieve several columns from the original layer query.
Definition
```
{
// REQUIRED
// string, `type` the list type
“type”: “list”,
// REQUIRED
// object, `options` dataview params
“options”: {
// REQUIRED
// array, `columns` to select for the list
“columns”: [“name”, “description”]
}
}
```
Expected output
```
{
"type": "list",
"rows": [
{
"{columnName1}": "val1",
"{columnName2}": 100
},
{
"{columnName1}": "val2",
"{columnName2}": 200
}
]
}
```
### Aggregation
An aggregation is very similar to a list but results are aggregated by a column and a given aggregation function.
An aggregation is a list with aggregated results by a column and a given aggregation function.
Definition
```

8
lib/cartodb/controllers/layergroup.js
View File

@ -196,9 +196,9 @@ LayergroupController.prototype.dataview = function(req, res, next) {
self.mapStore, res.locals.user, self.userLimitsApi, res.locals
);
self.dataviewBackend.getDataview(
mapConfigProvider,
res.locals.user,
res.locals,
mapConfigProvider,
res.locals.user,
res.locals,
this
);
},
@ -343,7 +343,7 @@ LayergroupController.prototype.bbox = function(req, res, next) {
north: +req.params.north,
east: +req.params.east,
south: +req.params.south
}, next);
}, null, next);
};
LayergroupController.prototype.center = function(req, res, next) {

635
lib/cartodb/models/dataview/aggregation.js
View File

@ -1,95 +1,178 @@
var _ = require('underscore');
var BaseWidget = require('./base');
var debug = require('debug')('windshaft:widget:aggregation');
const BaseDataview = require('./base');
const debug = require('debug')('windshaft:dataview:aggregation');
var dot = require('dot');
dot.templateSettings.strip = false;
const filteredQueryTpl = ctx => `
filtered_source AS (
SELECT *
FROM (${ctx.query}) _cdb_filtered_source
${ctx.aggregationColumn && ctx.isFloatColumn ? `
WHERE
${ctx.aggregationColumn} != 'infinity'::float
AND
${ctx.aggregationColumn} != '-infinity'::float
AND
${ctx.aggregationColumn} != 'NaN'::float` :
''
}
)
`;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
const summaryQueryTpl = ctx => `
summary AS (
SELECT
count(1) AS count,
sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count
${ctx.isFloatColumn ? `,
sum(
CASE
WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float
THEN 1
ELSE 0
END
) AS infinities_count,
sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` :
''
}
FROM (${ctx.query}) _cdb_aggregation_nulls
)
`;
var summaryQueryTpl = dot.template([
'summary AS (',
' SELECT',
' count(1) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' {{?it._isFloatColumn}},sum(',
' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1',
' ELSE 0',
' END',
' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')'
].join('\n'));
const rankedCategoriesQueryTpl = ctx => `
categories AS(
SELECT
${ctx.column} AS category,
${ctx.aggregationFn} AS value,
row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
FROM filtered_source
${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
GROUP BY ${ctx.column}
ORDER BY 2 DESC
)
`;
var rankedCategoriesQueryTpl = dot.template([
'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM filtered_source',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
')'
].join('\n'));
const categoriesSummaryMinMaxQueryTpl = () => `
categories_summary_min_max AS(
SELECT
max(value) max_val,
min(value) min_val
FROM categories
)
`;
var categoriesSummaryMinMaxQueryTpl = dot.template([
'categories_summary_min_max AS(',
' SELECT max(value) max_val, min(value) min_val',
' FROM categories',
')'
].join('\n'));
const categoriesSummaryCountQueryTpl = ctx => `
categories_summary_count AS(
SELECT count(1) AS categories_count
FROM (
SELECT ${ctx.column} AS category
FROM (${ctx.query}) _cdb_categories
GROUP BY ${ctx.column}
) _cdb_categories_count
)
`;
var categoriesSummaryCountQueryTpl = dot.template([
'categories_summary_count AS(',
' SELECT count(1) AS categories_count',
' FROM (',
' SELECT {{=it._column}} AS category',
' FROM ({{=it._query}}) _cdb_categories',
' GROUP BY {{=it._column}}',
' ) _cdb_categories_count',
')'
].join('\n'));
const specialNumericValuesColumns = () => `, nans_count, infinities_count`;
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,',
' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n'));
const rankedAggregationQueryTpl = ctx => `
SELECT
CAST(category AS text),
value,
false as agg,
nulls_count,
min_val,
max_val,
count,
categories_count
${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
FROM categories, summary, categories_summary_min_max, categories_summary_count
WHERE rank < ${ctx.limit}
UNION ALL
SELECT
'Other' category,
${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value,
true as agg,
nulls_count,
min_val,
max_val,
count,
categories_count
${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
FROM categories, summary, categories_summary_min_max, categories_summary_count
WHERE rank >= ${ctx.limit}
GROUP BY
nulls_count,
min_val,
max_val,
count,
categories_count
${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
`;
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC'
].join('\n'));
const aggregationQueryTpl = ctx => `
SELECT
CAST(${ctx.column} AS text) AS category,
${ctx.aggregationFn} AS value,
false as agg,
nulls_count,
min_val,
max_val,
count,
categories_count
${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count
GROUP BY
category,
nulls_count,
min_val,
max_val,
count,
categories_count
${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
ORDER BY value DESC
`;
var CATEGORIES_LIMIT = 6;
const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`;
var VALID_OPERATIONS = {
const aggregationDataviewQueryTpl = ctx => `
WITH
${filteredQueryTpl(ctx)},
${summaryQueryTpl(ctx)},
${rankedCategoriesQueryTpl(ctx)},
${categoriesSummaryMinMaxQueryTpl(ctx)},
${categoriesSummaryCountQueryTpl(ctx)}
${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`}
`;
const filterCategoriesQueryTpl = ctx => `
SELECT
${ctx.column} AS category,
${ctx.value} AS value
FROM (${ctx.query}) _cdb_aggregation_search
WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery}
GROUP BY ${ctx.column}
`;
const searchQueryTpl = ctx => `
WITH
search_unfiltered AS (
${ctx.searchUnfiltered}
),
search_filtered AS (
${ctx.searchFiltered}
),
search_union AS (
SELECT * FROM search_unfiltered
UNION ALL
SELECT * FROM search_filtered
)
SELECT category, sum(value) AS value
FROM search_union
GROUP BY category
ORDER BY value desc
`;
const CATEGORIES_LIMIT = 6;
const VALID_OPERATIONS = {
count: [],
sum: ['aggregationColumn'],
avg: ['aggregationColumn'],
@ -97,7 +180,7 @@ var VALID_OPERATIONS = {
max: ['aggregationColumn']
};
var TYPE = 'aggregation';
const TYPE = 'aggregation';
/**
{
@ -108,256 +191,150 @@ var TYPE = 'aggregation';
}
}
*/
function Aggregation(query, options, queries) {
if (!_.isString(options.column)) {
throw new Error('Aggregation expects `column` in widget options');
module.exports = class Aggregation extends BaseDataview {
constructor (query, options = {}, queries = {}) {
super();
this._checkOptions(options);
this.query = query;
this.queries = queries;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
}
if (!_.isString(options.aggregation)) {
throw new Error('Aggregation expects `aggregation` operation in widget options');
}
if (!VALID_OPERATIONS[options.aggregation]) {
throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
}
var requiredOptions = VALID_OPERATIONS[options.aggregation];
var missingOptions = _.difference(requiredOptions, Object.keys(options));
if (missingOptions.length > 0) {
throw new Error(
"Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
);
}
BaseWidget.apply(this);
this.query = query;
this.queries = queries;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
}
Aggregation.prototype = new BaseWidget();
Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var _query = this.query;
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
_limit: CATEGORIES_LIMIT
})
].join('\n');
} else {
aggregationSql = [
this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
_limit: CATEGORIES_LIMIT
})
].join('\n');
}
debug(aggregationSql);
return callback(null, aggregationSql);
};
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) {
return [
"WITH",
[
filteredQueryTpl({
_isFloatColumn: isFloatColumn,
_query: this.query,
_column: this.column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
summaryQueryTpl({
_isFloatColumn: isFloatColumn,
_query: query,
_column: column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
rankedCategoriesQueryTpl({
_query: query,
_column: column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
categoriesSummaryMinMaxQueryTpl({
_query: query,
_column: column
}),
categoriesSummaryCountQueryTpl({
_query: query,
_column: column
})
].join(',\n')
].join('\n');
};
var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
Aggregation.prototype.getAggregationSql = function() {
return aggregationFnQueryTpl({
_aggregationFn: this.aggregation,
_aggregationColumn: this.aggregationColumn || 1
});
};
Aggregation.prototype.format = function(result) {
var categories = [];
var count = 0;
var nulls = 0;
var nans = 0;
var infinities = 0;
var minValue = 0;
var maxValue = 0;
var categoriesCount = 0;
if (result.rows.length) {
var firstRow = result.rows[0];
count = firstRow.count;
nulls = firstRow.nulls_count;
nans = firstRow.nans_count;
infinities = firstRow.infinities_count;
minValue = firstRow.min_val;
maxValue = firstRow.max_val;
categoriesCount = firstRow.categories_count;
result.rows.forEach(function(row) {
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val',
'max_val', 'categories_count', 'nans_count', 'infinities_count'));
});
}
return {
aggregation: this.aggregation,
count: count,
nulls: nulls,
nans: nans,
infinities: infinities,
min: minValue,
max: maxValue,
categoriesCount: categoriesCount,
categories: categories
};
};
var filterCategoriesQueryTpl = dot.template([
'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
'FROM ({{=it._query}}) _cdb_aggregation_search',
'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
'GROUP BY {{=it._column}}'
].join('\n'));
var searchQueryTpl = dot.template([
'WITH',
'search_unfiltered AS (',
' {{=it._searchUnfiltered}}',
'),',
'search_filtered AS (',
' {{=it._searchFiltered}}',
'),',
'search_union AS (',
' SELECT * FROM search_unfiltered',
' UNION ALL',
' SELECT * FROM search_filtered',
')',
'SELECT category, sum(value) AS value',
'FROM search_union',
'GROUP BY category',
'ORDER BY value desc'
].join('\n'));
Aggregation.prototype.search = function(psql, userQuery, callback) {
var self = this;
var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
var _value = this.aggregation !== 'count' && this.aggregationColumn ?
this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)';
// TODO unfiltered will be wrong as filters are already applied at this point
var query = searchQueryTpl({
_searchUnfiltered: filterCategoriesQueryTpl({
_query: this.query,
_column: this.column,
_value: '0',
_userQuery: _userQuery
}),
_searchFiltered: filterCategoriesQueryTpl({
_query: this.query,
_column: this.column,
_value: _value,
_userQuery: _userQuery
})
});
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
_checkOptions (options) {
if (typeof options.column !== 'string') {
throw new Error(`Aggregation expects 'column' in dataview options`);
}
return callback(null, {type: self.getType(), categories: result.rows });
}, true); // use read-only transaction
};
if (typeof options.aggregation !== 'string') {
throw new Error(`Aggregation expects 'aggregation' operation in dataview options`);
}
Aggregation.prototype.getType = function() {
return TYPE;
};
if (!VALID_OPERATIONS[options.aggregation]) {
throw new Error(`Aggregation does not support '${options.aggregation}' operation`);
}
Aggregation.prototype.toString = function() {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_column: this.column,
_aggregation: this.aggregation
});
const requiredOptions = VALID_OPERATIONS[options.aggregation];
const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption));
if (missingOptions.length > 0) {
throw new Error(
`Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`
);
}
}
sql (psql, override, callback) {
if (!callback) {
callback = override;
override = {};
}
if (this._shouldCheckColumnType()) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => {
if (!err && !!type) {
this._isFloatColumn = type.float;
}
this.sql(psql, override, callback);
});
return null;
}
const aggregationSql = aggregationDataviewQueryTpl({
override: override,
query: this.query,
column: this.column,
aggregation: this.aggregation,
aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null,
aggregationFn: aggregationFnQueryTpl({
aggregation: this.aggregation,
aggregationColumn: this.aggregationColumn || 1
}),
isFloatColumn: this._isFloatColumn,
limit: CATEGORIES_LIMIT
});
debug(aggregationSql);
return callback(null, aggregationSql);
}
_shouldCheckColumnType () {
return this.aggregationColumn && this._isFloatColumn === null;
}
format (result) {
const {
count = 0,
nulls_count = 0,
nans_count = 0,
infinities_count = 0,
min_val = 0,
max_val = 0,
categories_count = 0
} = result.rows[0] || {};
return {
aggregation: this.aggregation,
count: count,
nulls: nulls_count,
nans: nans_count,
infinities: infinities_count,
min: min_val,
max: max_val,
categoriesCount: categories_count,
categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg }))
};
}
search (psql, userQuery, callback) {
const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`);
const value = this.aggregation !== 'count' && this.aggregationColumn ?
`${this.aggregation}(${this.aggregationColumn})` :
'count(1)';
// TODO unfiltered will be wrong as filters are already applied at this point
const query = searchQueryTpl({
searchUnfiltered: filterCategoriesQueryTpl({
query: this.query,
column: this.column,
value: '0',
userQuery: escapedUserQuery
}),
searchFiltered: filterCategoriesQueryTpl({
query: this.query,
column: this.column,
value: value,
userQuery: escapedUserQuery
})
});
debug(query);
psql.query(query, (err, result) => {
if (err) {
return callback(err, result);
}
return callback(null, {type: this.getType(), categories: result.rows });
}, true); // use read-only transaction
}
getType () {
return TYPE;
}
toString () {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_column: this.column,
_aggregation: this.aggregation
});
}
};

96
lib/cartodb/models/dataview/base.js
View File

@ -1,67 +1,16 @@
var dot = require('dot');
dot.templateSettings.strip = false;
function BaseDataview() {}
module.exports = BaseDataview;
BaseDataview.prototype.getResult = function(psql, override, callback) {
var self = this;
this.sql(psql, override, function(err, query) {
if (err) {
return callback(err);
}
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
}
result = self.format(result, override);
result.type = self.getType();
return callback(null, result);
}, true); // use read-only transaction
});
};
BaseDataview.prototype.search = function(psql, userQuery, callback) {
return callback(null, this.format({ rows: [] }));
};
var FLOAT_OIDS = {
const FLOAT_OIDS = {
700: true,
701: true,
1700: true
};
var DATE_OIDS = {
const DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1'
);
BaseDataview.prototype.getColumnType = function (psql, column, query, callback) {
var readOnlyTransaction = true;
var columnTypeQuery = columnTypeQueryTpl({
column: column, query: query
});
psql.query(columnTypeQuery, function(err, result) {
if (err) {
return callback(err);
}
var pgType = result.rows[0].pg_typeof;
callback(null, getPGTypeName(pgType));
}, readOnlyTransaction);
};
const columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`;
function getPGTypeName (pgType) {
return {
@ -69,3 +18,42 @@ function getPGTypeName (pgType) {
date: DATE_OIDS.hasOwnProperty(pgType)
};
}
module.exports = class BaseDataview {
getResult (psql, override, callback) {
this.sql(psql, override, (err, query) => {
if (err) {
return callback(err);
}
psql.query(query, (err, result) => {
if (err) {
return callback(err, result);
}
result = this.format(result, override);
result.type = this.getType();
return callback(null, result);
}, true); // use read-only transaction
});
}
search (psql, userQuery, callback) {
return callback(null, this.format({ rows: [] }));
}
getColumnType (psql, column, query, callback) {
const readOnlyTransaction = true;
const columnTypeQuery = columnTypeQueryTpl({ column, query });
psql.query(columnTypeQuery, (err, result) => {
if (err) {
return callback(err);
}
const pgType = result.rows[0].pg_typeof;
callback(null, getPGTypeName(pgType));
}, readOnlyTransaction);
}
};

24
lib/cartodb/models/dataview/factory.js
View File

@ -1,18 +1,20 @@
var dataviews = require('./');
const dataviews = require('./');
var DataviewFactory = {
dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
return allDataviews;
}, {}),
module.exports = class DataviewFactory {
static get dataviews() {
return Object.keys(dataviews).reduce((allDataviews, dataviewClassName) => {
allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
return allDataviews;
}, {});
}
static getDataview (query, dataviewDefinition) {
const { type, options, sql } = dataviewDefinition;
getDataview: function(query, dataviewDefinition) {
var type = dataviewDefinition.type;
if (!this.dataviews[type]) {
throw new Error('Invalid dataview type: "' + type + '"');
}
return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql);
return new this.dataviews[type](query, options, sql);
}
};
module.exports = DataviewFactory;

201
lib/cartodb/models/dataview/formula.js
View File

@ -1,28 +1,36 @@
var _ = require('underscore');
var BaseWidget = require('./base');
var debug = require('debug')('windshaft:widget:formula');
const BaseDataview = require('./base');
const debug = require('debug')('windshaft:dataview:formula');
var dot = require('dot');
dot.templateSettings.strip = false;
const countInfinitiesQueryTpl = ctx => `
SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities
WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float
`;
var formulaQueryTpl = dot.template([
'SELECT',
' {{=it._operation}}({{=it._column}}) AS result,',
' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n'));
const countNansQueryTpl = ctx => `
SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans
WHERE ${ctx.column} = 'NaN'::float
`;
var VALID_OPERATIONS = {
const filterOutSpecialNumericValuesTpl = ctx => `
WHERE
${ctx.column} != 'infinity'::float
AND
${ctx.column} != '-infinity'::float
AND
${ctx.column} != 'NaN'::float
`;
const formulaQueryTpl = ctx => `
SELECT
${ctx.operation}(${ctx.column}) AS result,
(SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count
${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''}
${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''}
FROM (${ctx.query}) __cdb_formula
${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''}
`;
const VALID_OPERATIONS = {
count: true,
avg: true,
sum: true,
@ -30,7 +38,7 @@ var VALID_OPERATIONS = {
max: true
};
var TYPE = 'formula';
const TYPE = 'formula';
/**
{
@ -41,93 +49,90 @@ var TYPE = 'formula';
}
}
*/
function Formula(query, options, queries) {
if (!_.isString(options.operation)) {
throw new Error('Formula expects `operation` in widget options');
module.exports = class Formula extends BaseDataview {
constructor (query, options = {}, queries = {}) {
super();
this._checkOptions(options);
this.query = query;
this.queries = queries;
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
}
if (!VALID_OPERATIONS[options.operation]) {
throw new Error("Formula does not support '" + options.operation + "' operation");
_checkOptions (options) {
if (typeof options.operation !== 'string') {
throw new Error(`Formula expects 'operation' in dataview options`);
}
if (!VALID_OPERATIONS[options.operation]) {
throw new Error(`Formula does not support '${options.operation}' operation`);
}
if (options.operation !== 'count' && typeof options.column !== 'string') {
throw new Error(`Formula expects 'column' in dataview options`);
}
}
if (options.operation !== 'count' && !_.isString(options.column)) {
throw new Error('Formula expects `column` in widget options');
}
BaseWidget.apply(this);
sql (psql, override, callback) {
if (!callback) {
callback = override;
override = {};
}
this.query = query;
this.queries = queries;
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
}
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
if (!err && !!type) {
this._isFloatColumn = type.float;
}
this.sql(psql, override, callback);
});
return null;
}
Formula.prototype = new BaseWidget();
Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
const formulaSql = formulaQueryTpl({
isFloatColumn: this._isFloatColumn,
query: this.query,
operation: this.operation,
column: this.column
});
return null;
debug(formulaSql);
return callback(null, formulaSql);
}
var formulaSql = formulaQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: this.query,
_operation: this.operation,
_column: this.column
});
format (res) {
const {
result = 0,
nulls_count = 0,
nans_count,
infinities_count
} = res.rows[0] || {};
debug(formulaSql);
return callback(null, formulaSql);
};
Formula.prototype.format = function(result) {
var formattedResult = {
operation: this.operation,
result: 0,
nulls: 0,
nans: 0,
infinities: 0
};
if (result.rows.length) {
formattedResult.operation = this.operation;
formattedResult.result = result.rows[0].result;
formattedResult.nulls = result.rows[0].nulls_count;
formattedResult.nans = result.rows[0].nans_count;
formattedResult.infinities = result.rows[0].infinities_count;
return {
operation: this.operation,
result,
nulls: nulls_count,
nans: nans_count,
infinities: infinities_count
};
}
return formattedResult;
};
getType () {
return TYPE;
}
Formula.prototype.getType = function() {
return TYPE;
};
Formula.prototype.toString = function() {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_column: this.column,
_operation: this.operation
});
toString () {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_column: this.column,
_operation: this.operation
});
}
};

761
lib/cartodb/models/dataview/histogram.js
View File

@ -1,717 +1,72 @@
var _ = require('underscore');
var BaseWidget = require('./base');
var debug = require('debug')('windshaft:dataview:histogram');
const debug = require('debug')('windshaft:dataview:histogram');
const NumericHistogram = require('./histograms/numeric-histogram');
const DateHistogram = require('./histograms/date-histogram');
var dot = require('dot');
dot.templateSettings.strip = false;
const DATE_HISTOGRAM = 'DateHistogram';
const NUMERIC_HISTOGRAM = 'NumericHistogram';
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
module.exports = class Histogram {
constructor (query, options, queries) {
this.query = query;
this.options = options || {};
this.queries = queries;
var dateIntervalQueryTpl = dot.template([
'WITH',
'__cdb_dates AS (',
' SELECT',
' MAX({{=it.column}}::timestamp) AS __cdb_end,',
' MIN({{=it.column}}::timestamp) AS __cdb_start',
' FROM ({{=it.query}}) __cdb_source',
'),',
'__cdb_interval_in_days AS (',
' SELECT' ,
' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days',
' FROM __cdb_dates',
'),',
'__cdb_interval_in_hours AS (',
' SELECT',
' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours',
' FROM __cdb_interval_in_days, __cdb_dates',
'),',
'__cdb_interval_in_minutes AS (',
' SELECT',
' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes',
' FROM __cdb_interval_in_hours, __cdb_dates',
'),',
'__cdb_interval_in_seconds AS (',
' SELECT',
' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds',
' FROM __cdb_interval_in_minutes, __cdb_dates',
')',
'SELECT',
' ROUND(__cdb_days / 365) AS year,',
' ROUND(__cdb_days / 90) AS quarter,',
' ROUND(__cdb_days / 30) AS month,',
' ROUND(__cdb_days / 7) AS week,',
' __cdb_days AS day,',
' __cdb_hours AS hour,',
' __cdb_minutes AS minute,',
' __cdb_seconds AS second',
'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds'
].join('\n'));
var MAX_INTERVAL_VALUE = 366;
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
var filteredQueryTpl = dot.template([
'__cdb_filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) __cdb_filtered_source_query',
' WHERE',
' {{=it._column}} IS NOT NULL',
' {{?it._isFloatColumn}}AND',
' {{=it._column}} != \'infinity\'::float',
' AND',
' {{=it._column}} != \'-infinity\'::float',
' AND',
' {{=it._column}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var basicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,',
' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
' FROM __cdb_filtered_source',
')'
].join(' \n'));
var overrideBasicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,',
' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
' FROM __cdb_filtered_source',
')'
].join('\n'));
var iqrQueryTpl = dot.template([
'__cdb_iqrange AS (',
' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr',
' FROM (',
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile',
' FROM __cdb_filtered_source) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile',
' ) __cdb_iqr',
')'
].join('\n'));
var binsQueryTpl = dot.template([
'__cdb_bins AS (',
' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0',
' THEN 1',
' ELSE GREATEST(',
' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),',
' LEAST(',
' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),',
' {{=it._maxBins}}',
' )',
' )',
' END AS __cdb_bins_number',
' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source',
' LIMIT 1',
')'
].join('\n'));
var overrideBinsQueryTpl = dot.template([
'__cdb_bins AS (',
' SELECT {{=it._bins}} AS __cdb_bins_number',
')'
].join('\n'));
var nullsQueryTpl = dot.template([
'__cdb_nulls AS (',
' SELECT',
' count(*) AS __cdb_nulls_count',
' FROM ({{=it._query}}) __cdb_histogram_nulls',
' WHERE {{=it._column}} IS NULL',
')'
].join('\n'));
var infinitiesQueryTpl = dot.template([
'__cdb_infinities AS (',
' SELECT',
' count(*) AS __cdb_infinities_count',
' FROM ({{=it._query}}) __cdb_infinities_query',
' WHERE',
' {{=it._column}} = \'infinity\'::float',
' OR',
' {{=it._column}} = \'-infinity\'::float',
')'
].join('\n'));
var nansQueryTpl = dot.template([
'__cdb_nans AS (',
' SELECT',
' count(*) AS __cdb_nans_count',
' FROM ({{=it._query}}) __cdb_nans_query',
' WHERE {{=it._column}} = \'NaN\'::float',
')'
].join('\n'));
var histogramQueryTpl = dot.template([
'SELECT',
' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
' __cdb_bins_number AS bins_number,',
' __cdb_nulls_count AS nulls_count,',
' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,',
' __cdb_nans_count AS nans_count,{{?}}',
' __cdb_avg_val AS avg_val,',
' CASE WHEN __cdb_min_val = __cdb_max_val',
' THEN 0',
' ELSE GREATEST(',
' 1,',
' LEAST(',
' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),',
' __cdb_bins_number',
' )',
' ) - 1',
' END AS bin,',
' min({{=it._column}})::numeric AS min,',
' max({{=it._column}})::numeric AS max,',
' avg({{=it._column}})::numeric AS avg,',
' count(*) AS freq',
'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,',
' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}',
'GROUP BY bin, bins_number, bin_width, nulls_count,',
' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
'ORDER BY bin'
].join('\n'));
var dateBasicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,',
' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
' min(date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' )) AS __cdb_start_date,',
' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,',
' count(1) AS __cdb_total_rows',
' FROM ({{=it._query}}) __cdb_basics_query',
')'
].join(' \n'));
var dateOverrideBasicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max({{=it._end}})::float AS __cdb_max_val,',
' min({{=it._start}})::float AS __cdb_min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
' min(',
' date_trunc(',
' \'{{=it._aggregation}}\',',
' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' )',
' ) AS __cdb_start_date,',
' max(',
' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' ) AS __cdb_end_date,',
' count(1) AS __cdb_total_rows',
' FROM ({{=it._query}}) __cdb_basics_query',
')'
].join(' \n'));
var dateBinsQueryTpl = dot.template([
'__cdb_bins AS (',
' SELECT',
' __cdb_bins_array,',
' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number',
' FROM (',
' SELECT',
' ARRAY(',
' SELECT GENERATE_SERIES(',
' __cdb_start_date::timestamptz,',
' __cdb_end_date::timestamptz,',
' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval',
' )',
' ) AS __cdb_bins_array',
' FROM __cdb_basics',
' ) __cdb_bins_array_query',
')'
].join('\n'));
var dateHistogramQueryTpl = dot.template([
'SELECT',
' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
' __cdb_bins_number AS bins_number,',
' __cdb_nulls_count AS nulls_count,',
' CASE WHEN __cdb_min_val = __cdb_max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(',
' WIDTH_BUCKET(',
' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',',
' __cdb_bins_array',
' ),',
' __cdb_bins_number',
' )) - 1',
' END AS bin,',
' min(',
' date_part(',
' \'epoch\', ',
' date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' ) AT TIME ZONE \'{{=it._offset}}\'',
' )',
' )::numeric AS timestamp,',
' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,',
' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
' count(*) AS freq',
'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls',
'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start',
'ORDER BY bin'
].join('\n'));
var TYPE = 'histogram';
/**
Numeric histogram:
{
type: 'histogram',
options: {
column: 'name', // column data type: numeric
bins: 10 // OPTIONAL
}
}
Time series:
{
type: 'histogram',
options: {
column: 'date', // column data type: date
aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric)
offset: -7200 // OPTIONAL (UTC offset in seconds)
}
}
*/
function Histogram(query, options, queries) {
if (!_.isString(options.column)) {
throw new Error('Histogram expects `column` in widget options');
this.histogramImplementation = this._getHistogramImplementation();
}
this.query = query;
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
this.aggregation = options.aggregation;
this.offset = options.offset;
_getHistogramImplementation (override) {
let implementation = null;
this._columnType = null;
}
Histogram.prototype = new BaseWidget();
Histogram.prototype.constructor = Histogram;
module.exports = Histogram;
Histogram.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
if (this._columnType === null) {
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
// assume numeric, will fail later
self._columnType = 'numeric';
if (!err && !!type) {
self._columnType = Object.keys(type).find(function (key) {
return type[key];
});
}
self.sql(psql, override, callback);
}, true); // use read-only transaction
return null;
}
this._buildQuery(psql, override, callback);
};
Histogram.prototype.isDateHistogram = function (override) {
return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined);
};
Histogram.prototype._buildQuery = function (psql, override, callback) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.query;
if (this.isDateHistogram(override)) {
return this._buildDateHistogramQuery(psql, override, callback);
}
if (this._columnType === 'date') {
_column = columnCastTpl({column: _column});
}
filteredQuery = filteredQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
});
if (this._shouldOverride(override)) {
debug('overriding with %j', override);
basicsQuery = overrideBasicsQueryTpl({
_query: _query,
_column: _column,
_start: getBinStart(override),
_end: getBinEnd(override)
});
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
})
].join(',\n');
} else {
basicsQuery = basicsQueryTpl({
_query: _query,
_column: _column
});
if (this._shouldOverrideBins(override)) {
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
})
].join(',\n');
} else {
binsQuery = [
iqrQueryTpl({
_query: _query,
_column: _column
}),
binsQueryTpl({
_query: _query,
_minBins: BIN_MIN_NUMBER,
_maxBins: BIN_MAX_NUMBER
})
].join(',\n');
}
}
var cteSql = [
filteredQuery,
basicsQuery,
binsQuery,
nullsQueryTpl({
_query: _query,
_column: _column
})
];
if (this._columnType === 'float') {
cteSql.push(
infinitiesQueryTpl({
_query: _query,
_column: _column
}),
nansQueryTpl({
_query: _query,
_column: _column
})
);
}
var histogramSql = [
"WITH",
cteSql.join(',\n'),
histogramQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
})
].join('\n');
debug(histogramSql);
return callback(null, histogramSql);
};
Histogram.prototype._shouldOverride = function (override) {
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
};
Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};
var DATE_AGGREGATIONS = {
'auto': true,
'minute': true,
'hour': true,
'day': true,
'week': true,
'month': true,
'quarter': true,
'year': true
};
Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) {
var _column = this.column;
var _query = this.query;
var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset;
if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) {
return callback(new Error('Invalid aggregation value. Valid ones: ' +
Object.keys(DATE_AGGREGATIONS).join(', ')
));
}
if (_aggregation === 'auto') {
this.getAutomaticAggregation(psql, function (err, aggregation) {
if (err || aggregation === 'none') {
this.aggregation = 'day';
} else {
this.aggregation = aggregation;
}
override.aggregation = this.aggregation;
this._buildDateHistogramQuery(psql, override, callback);
}.bind(this));
return null;
}
var dateBasicsQuery;
if (override && _.has(override, 'start') && _.has(override, 'end')) {
dateBasicsQuery = dateOverrideBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_start: getBinStart(override),
_end: getBinEnd(override),
_offset: parseOffset(_offset, _aggregation)
});
} else {
dateBasicsQuery = dateBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
}
var dateBinsQuery = [
dateBinsQueryTpl({
_aggregation: _aggregation
})
].join(',\n');
var nullsQuery = nullsQueryTpl({
_query: _query,
_column: _column
});
var dateHistogramQuery = dateHistogramQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
var histogramSql = [
"WITH",
[
dateBasicsQuery,
dateBinsQuery,
nullsQuery
].join(',\n'),
dateHistogramQuery
].join('\n');
debug(histogramSql);
return callback(null, histogramSql);
};
Histogram.prototype.getAutomaticAggregation = function (psql, callback) {
var dateIntervalQuery = dateIntervalQueryTpl({
query: this.query,
column: this.column
});
debug(dateIntervalQuery);
psql.query(dateIntervalQuery, function (err, result) {
if (err) {
return callback(err);
switch (this._getHistogramSubtype(override)) {
case DATE_HISTOGRAM:
debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override);
implementation = new DateHistogram(this.query, this.options, this.queries);
break;
case NUMERIC_HISTOGRAM:
debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override);
implementation = new NumericHistogram(this.query, this.options, this.queries);
break;
default:
throw new Error('Unsupported Histogram type');
}
var aggegations = result.rows[0];
var aggregation = Object.keys(aggegations)
.map(function (key) {
return {
name: key,
value: aggegations[key]
};
})
.reduce(function (closer, current) {
if (current.value > MAX_INTERVAL_VALUE) {
return closer;
}
return implementation;
}
var closerDiff = MAX_INTERVAL_VALUE - closer.value;
var currentDiff = MAX_INTERVAL_VALUE - current.value;
if (Number.isFinite(current.value) && closerDiff > currentDiff) {
return current;
}
return closer;
}, { name: 'none', value: -1 });
callback(null, aggregation.name);
});
};
Histogram.prototype.format = function(result, override) {
override = override || {};
var buckets = [];
var binsCount = getBinsCount(override);
var width = getWidth(override);
var binsStart = getBinStart(override);
var nulls = 0;
var infinities = 0;
var nans = 0;
var avg;
var timestampStart;
var aggregation;
var offset;
if (result.rows.length) {
var firstRow = result.rows[0];
binsCount = firstRow.bins_number;
width = firstRow.bin_width || width;
avg = firstRow.avg_val;
nulls = firstRow.nulls_count;
timestampStart = firstRow.timestamp_start;
infinities = firstRow.infinities_count;
nans = firstRow.nans_count;
binsStart = populateBinStart(override, firstRow);
if (Number.isFinite(timestampStart)) {
aggregation = getAggregation(override, this.aggregation);
offset = getOffset(override, this.offset);
_getHistogramSubtype (override) {
if(this._isDateHistogram(override)) {
return DATE_HISTOGRAM;
}
buckets = result.rows.map(function(row) {
return _.omit(
row,
'bins_number',
'bin_width',
'nulls_count',
'infinities_count',
'nans_count',
'avg_val',
'timestamp_start'
);
});
return NUMERIC_HISTOGRAM;
}
return {
aggregation: aggregation,
offset: offset,
timestamp_start: timestampStart,
bin_width: width,
bins_count: binsCount,
bins_start: binsStart,
nulls: nulls,
infinities: infinities,
nans: nans,
avg: avg,
bins: buckets
};
};
function getAggregation(override, aggregation) {
return override && override.aggregation ? override.aggregation : aggregation;
}
function getOffset(override, offset) {
if (override && override.offset) {
return override.offset;
}
if (offset) {
return offset;
}
return 0;
}
function getBinStart(override) {
if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
return Math.min(override.start, override.end);
}
return override.start || 0;
}
function getBinEnd(override) {
if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
return Math.max(override.start, override.end);
}
return override.end || 0;
}
function getBinsCount(override) {
return override.bins || 0;
}
function getWidth(override) {
var width = 0;
var binsCount = override.bins;
if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) {
width = (override.end - override.start) / binsCount;
}
return width;
}
function parseOffset(offset, aggregation) {
if (!offset) {
return '0';
}
if (aggregation === 'hour' || aggregation === 'minute') {
return '0';
}
var offsetInHours = Math.ceil(offset / 3600);
return '' + offsetInHours;
}
function populateBinStart(override, firstRow) {
var binStart;
if (firstRow.hasOwnProperty('timestamp')) {
binStart = firstRow.timestamp;
} else if (override.hasOwnProperty('start')) {
binStart = getBinStart(override);
} else {
binStart = firstRow.min;
}
return binStart;
}
Histogram.prototype.getType = function() {
return TYPE;
};
Histogram.prototype.toString = function() {
return JSON.stringify({
_type: TYPE,
_column: this.column,
_query: this.query
});
_isDateHistogram (override = {}) {
return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation'));
}
getResult (psql, override, callback) {
this.histogramImplementation = this._getHistogramImplementation(override);
this.histogramImplementation.getResult(psql, override, callback);
}
// In order to keep previous behaviour with overviews,
// we have to expose the following methods to bypass
// the concrete overview implementation
sql (psql, override, callback) {
this.histogramImplementation.sql(psql, override, callback);
}
format (result, override) {
return this.histogramImplementation.format(result, override);
}
getType () {
return this.histogramImplementation.getType();
}
toString () {
return this.histogramImplementation.toString();
}
};

85
lib/cartodb/models/dataview/histograms/base-histogram.js Normal file
View File

@ -0,0 +1,85 @@
const BaseDataview = require('../base');
const TYPE = 'histogram';
module.exports = class BaseHistogram extends BaseDataview {
constructor (query, options, queries) {
super();
if (typeof options.column !== 'string') {
throw new Error('Histogram expects `column` in widget options');
}
this.query = query;
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
this._columnType = null;
}
sql (psql, override, callback) {
if (!callback) {
callback = override;
override = {};
}
if (this._columnType === null) {
this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
// assume numeric, will fail later
this._columnType = 'numeric';
if (!err && !!type) {
this._columnType = Object.keys(type).find(function (key) {
return type[key];
});
}
this.sql(psql, override, callback);
}, true); // use read-only transaction
return null;
}
return this._buildQuery(psql, override, callback);
}
format (result, override) {
const histogram = this._getSummary(result, override);
histogram.bins = this._getBuckets(result);
return histogram;
}
getType () {
return TYPE;
}
toString () {
return JSON.stringify({
_type: TYPE,
_column: this.column,
_query: this.query
});
}
_hasOverridenRange (override) {
return override && override.hasOwnProperty('start') && override.hasOwnProperty('end');
}
_getBinStart (override = {}) {
if (this._hasOverridenRange(override)) {
return Math.min(override.start, override.end);
}
return override.start || 0;
}
_getBinEnd (override = {}) {
if (this._hasOverridenRange(override)) {
return Math.max(override.start, override.end);
}
return override.end || 0;
}
_getBinsCount (override = {}) {
return override.bins || 0;
}
};

302
lib/cartodb/models/dataview/histograms/date-histogram.js Normal file
View File

@ -0,0 +1,302 @@
const BaseHistogram = require('./base-histogram');
const debug = require('debug')('windshaft:dataview:date-histogram');
const dateIntervalQueryTpl = ctx => `
WITH
__cdb_dates AS (
SELECT
MAX(${ctx.column}::timestamp) AS __cdb_end,
MIN(${ctx.column}::timestamp) AS __cdb_start
FROM (${ctx.query}) __cdb_source
),
__cdb_interval_in_days AS (
SELECT
DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days
FROM __cdb_dates
),
__cdb_interval_in_hours AS (
SELECT
__cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours
FROM __cdb_interval_in_days, __cdb_dates
),
__cdb_interval_in_minutes AS (
SELECT
__cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes
FROM __cdb_interval_in_hours, __cdb_dates
),
__cdb_interval_in_seconds AS (
SELECT
__cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds
FROM __cdb_interval_in_minutes, __cdb_dates
)
SELECT
ROUND(__cdb_days / 365) AS year,
ROUND(__cdb_days / 90) AS quarter,
ROUND(__cdb_days / 30) AS month,
ROUND(__cdb_days / 7) AS week,
__cdb_days AS day,
__cdb_hours AS hour,
__cdb_minutes AS minute,
__cdb_seconds AS second
FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
`;
const nullsQueryTpl = ctx => `
__cdb_nulls AS (
SELECT
count(*) AS __cdb_nulls_count
FROM (${ctx.query}) __cdb_histogram_nulls
WHERE ${ctx.column} IS NULL
)
`;
const dateBasicsQueryTpl = ctx => `
__cdb_basics AS (
SELECT
max(date_part('epoch', ${ctx.column})) AS __cdb_max_val,
min(date_part('epoch', ${ctx.column})) AS __cdb_min_val,
avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
min(
date_trunc(
'${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
)
) AS __cdb_start_date,
max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date,
count(1) AS __cdb_total_rows
FROM (${ctx.query}) __cdb_basics_query
)
`;
const dateOverrideBasicsQueryTpl = ctx => `
__cdb_basics AS (
SELECT
max(${ctx.end})::float AS __cdb_max_val,
min(${ctx.start})::float AS __cdb_min_val,
avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
min(
date_trunc(
'${ctx.aggregation}',
TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}'
)
) AS __cdb_start_date,
max(
TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}'
) AS __cdb_end_date,
count(1) AS __cdb_total_rows
FROM (${ctx.query}) __cdb_basics_query
)
`;
const dateBinsQueryTpl = ctx => `
__cdb_bins AS (
SELECT
__cdb_bins_array,
ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number
FROM (
SELECT
ARRAY(
SELECT GENERATE_SERIES(
__cdb_start_date::timestamptz,
__cdb_end_date::timestamptz,
${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`}
)
) AS __cdb_bins_array
FROM __cdb_basics
) __cdb_bins_array_query
)
`;
const dateHistogramQueryTpl = ctx => `
SELECT
(__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
__cdb_bins_number AS bins_number,
__cdb_nulls_count AS nulls_count,
CASE WHEN __cdb_min_val = __cdb_max_val
THEN 0
ELSE GREATEST(
1,
LEAST(
WIDTH_BUCKET(
${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}',
__cdb_bins_array
),
__cdb_bins_number
)
) - 1
END AS bin,
min(
date_part(
'epoch',
date_trunc(
'${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
) AT TIME ZONE '${ctx.offset}'
)
)::numeric AS timestamp,
date_part('epoch', __cdb_start_date)::numeric AS timestamp_start,
min(date_part('epoch', ${ctx.column}))::numeric AS min,
max(date_part('epoch', ${ctx.column}))::numeric AS max,
avg(date_part('epoch', ${ctx.column}))::numeric AS avg,
count(*) AS freq
FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls
WHERE date_part('epoch', ${ctx.column}) IS NOT NULL
GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start
ORDER BY bin
`;
const MAX_INTERVAL_VALUE = 366;
const DATE_AGGREGATIONS = {
'auto': true,
'minute': true,
'hour': true,
'day': true,
'week': true,
'month': true,
'quarter': true,
'year': true
};
/**
date_histogram: {
type: 'histogram',
options: {
column: 'date', // column data type: date
aggregation: 'day' // MANDATORY
offset: -7200 // OPTIONAL (UTC offset in seconds)
}
}
*/
module.exports = class DateHistogram extends BaseHistogram {
constructor (query, options, queries) {
super(query, options, queries);
this.aggregation = options.aggregation;
this.offset = options.offset;
}
_buildQueryTpl (ctx) {
return `
WITH
${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)},
${dateBinsQueryTpl(ctx)},
${nullsQueryTpl(ctx)}
${dateHistogramQueryTpl(ctx)}
`;
}
_buildQuery (psql, override, callback) {
if (!this._isValidAggregation(override)) {
return callback(new Error('Invalid aggregation value. Valid ones: ' +
Object.keys(DATE_AGGREGATIONS).join(', ')
));
}
if (this._getAggregation(override) === 'auto') {
this._getAutomaticAggregation(psql, function (err, aggregation) {
if (err || aggregation === 'none') {
this.aggregation = 'day';
} else {
this.aggregation = aggregation;
}
override.aggregation = this.aggregation;
this._buildQuery(psql, override, callback);
}.bind(this));
return null;
}
const histogramSql = this._buildQueryTpl({
override: override,
query: this.query,
column: this.column,
aggregation: this._getAggregation(override),
start: this._getBinStart(override),
end: this._getBinEnd(override),
offset: this._parseOffset(override)
});
debug(histogramSql);
return callback(null, histogramSql);
}
_isValidAggregation (override) {
return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override));
}
_getAutomaticAggregation (psql, callback) {
const dateIntervalQuery = dateIntervalQueryTpl({
query: this.query,
column: this.column
});
psql.query(dateIntervalQuery, function (err, result) {
if (err) {
return callback(err);
}
const aggegations = result.rows[0];
const aggregation = Object.keys(aggegations)
.map(key => ({ name: key, value: aggegations[key] }))
.reduce((closer, current) => {
if (current.value > MAX_INTERVAL_VALUE) {
return closer;
}
const closerDiff = MAX_INTERVAL_VALUE - closer.value;
const currentDiff = MAX_INTERVAL_VALUE - current.value;
if (Number.isFinite(current.value) && closerDiff > currentDiff) {
return current;
}
return closer;
}, { name: 'none', value: -1 });
callback(null, aggregation.name);
});
}
_getSummary (result, override) {
const firstRow = result.rows[0] || {};
return {
aggregation: this._getAggregation(override),
offset: this._getOffset(override),
timestamp_start: firstRow.timestamp_start,
bin_width: firstRow.bin_width,
bins_count: firstRow.bins_number,
bins_start: firstRow.timestamp,
nulls: firstRow.nulls_count,
infinities: firstRow.infinities_count,
nans: firstRow.nans_count,
avg: firstRow.avg_val
};
}
_getBuckets (result) {
return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
}
_getAggregation (override = {}) {
return override.aggregation ? override.aggregation : this.aggregation;
}
_getOffset (override = {}) {
return Number.isFinite(override.offset) ? override.offset : (this.offset || 0);
}
_parseOffset (override) {
if (this._shouldIgnoreOffset(override)) {
return '0';
}
const offsetInHours = Math.ceil(this._getOffset(override) / 3600);
return '' + offsetInHours;
}
_shouldIgnoreOffset (override) {
return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute');
}
};

234
lib/cartodb/models/dataview/histograms/numeric-histogram.js Normal file
View File

@ -0,0 +1,234 @@
const BaseHistogram = require('./base-histogram');
const debug = require('debug')('windshaft:dataview:numeric-histogram');
const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`;
const filterOutSpecialNumericValues = ctx => `
${ctx.column} != 'infinity'::float
AND
${ctx.column} != '-infinity'::float
AND
${ctx.column} != 'NaN'::float
`;
const filteredQueryTpl = ctx => `
__cdb_filtered_source AS (
SELECT *
FROM (${ctx.query}) __cdb_filtered_source_query
WHERE ${ctx.column} IS NOT NULL
${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''}
)
`;
const basicsQueryTpl = ctx => `
__cdb_basics AS (
SELECT
max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val,
avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
FROM __cdb_filtered_source
)
`;
const overrideBasicsQueryTpl = ctx => `
__cdb_basics AS (
SELECT
max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val,
avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
FROM __cdb_filtered_source
)
`;
const iqrQueryTpl = ctx => `
__cdb_iqrange AS (
SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
FROM (
SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (
SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column}
) AS quartile
FROM __cdb_filtered_source) _cdb_quartiles
WHERE quartile = 1 or quartile = 3
GROUP BY quartile
) __cdb_iqr
)
`;
const binsQueryTpl = ctx => `
__cdb_bins AS (
SELECT
CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
THEN 1
ELSE GREATEST(
LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)),
LEAST(
CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),
${ctx.maxBins}
)
)
END AS __cdb_bins_number
FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source
LIMIT 1
)
`;
const overrideBinsQueryTpl = ctx => `
__cdb_bins AS (
SELECT ${ctx.override.bins} AS __cdb_bins_number
)
`;
const nullsQueryTpl = ctx => `
__cdb_nulls AS (
SELECT
count(*) AS __cdb_nulls_count
FROM (${ctx.query}) __cdb_histogram_nulls
WHERE ${ctx.column} IS NULL
)
`;
const infinitiesQueryTpl = ctx => `
__cdb_infinities AS (
SELECT
count(*) AS __cdb_infinities_count
FROM (${ctx.query}) __cdb_infinities_query
WHERE
${ctx.column} = 'infinity'::float
OR
${ctx.column} = '-infinity'::float
)
`;
const nansQueryTpl = ctx => `
__cdb_nans AS (
SELECT
count(*) AS __cdb_nans_count
FROM (${ctx.query}) __cdb_nans_query
WHERE ${ctx.column} = 'NaN'::float
)
`;
const specialNumericValuesColumnDefinitionTpl = () => `
__cdb_infinities_count AS infinities_count,
__cdb_nans_count AS nans_count
`;
const specialNumericValuesCTETpl = () => `
__cdb_infinities, __cdb_nans
`;
const specialNumericValuesColumnTpl = () => `
infinities_count, nans_count
`;
const histogramQueryTpl = ctx => `
SELECT
(__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
__cdb_bins_number AS bins_number,
__cdb_nulls_count AS nulls_count,
${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''}
__cdb_avg_val AS avg_val,
CASE WHEN __cdb_min_val = __cdb_max_val
THEN 0
ELSE GREATEST(
1,
LEAST(
WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),
__cdb_bins_number
)
) - 1
END AS bin,
min(${ctx.column})::numeric AS min,
max(${ctx.column})::numeric AS max,
avg(${ctx.column})::numeric AS avg,
count(*) AS freq
FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins
${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''}
GROUP BY bin, bins_number, bin_width, nulls_count, avg_val
${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''}
ORDER BY bin
`;
const BIN_MIN_NUMBER = 6;
const BIN_MAX_NUMBER = 48;
/**
Numeric histogram:
{
type: 'histogram',
options: {
column: 'name', // column data type: numeric
bins: 10 // OPTIONAL
}
}
*/
module.exports = class NumericHistogram extends BaseHistogram {
constructor (query, options, queries) {
super(query, options, queries);
}
_buildQuery (psql, override, callback) {
const histogramSql = this._buildQueryTpl({
override: override,
column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column,
isFloatColumn: this._columnType === 'float',
query: this.query,
start: this._getBinStart(override),
end: this._getBinEnd(override),
minBins: BIN_MIN_NUMBER,
maxBins: BIN_MAX_NUMBER,
});
debug(histogramSql);
return callback(null, histogramSql);
}
_buildQueryTpl (ctx) {
return `
WITH
${filteredQueryTpl(ctx)},
${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)},
${this._hasOverridenBins(ctx.override) ?
overrideBinsQueryTpl(ctx) :
`${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`
},
${nullsQueryTpl(ctx)}
${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''}
${histogramQueryTpl(ctx)}
`;
}
_hasOverridenBins (override) {
return override && override.hasOwnProperty('bins');
}
_getSummary (result, override) {
const firstRow = result.rows[0] || {};
return {
bin_width: firstRow.bin_width,
bins_count: firstRow.bins_number,
bins_start: this._populateBinStart(firstRow, override),
nulls: firstRow.nulls_count,
infinities: firstRow.infinities_count,
nans: firstRow.nans_count,
avg: firstRow.avg_val,
};
}
_getBuckets (result) {
return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq }));
}
_populateBinStart (firstRow, override = {}) {
let binStart;
if (override.hasOwnProperty('start')) {
binStart = this._getBinStart(override);
} else {
binStart = firstRow.min;
}
return binStart;
}
};

3
lib/cartodb/models/dataview/index.js
View File

@ -1,6 +1,5 @@
module.exports = {
Aggregation: require('./aggregation'),
Formula: require('./formula'),
Histogram: require('./histogram'),
List: require('./list')
Histogram: require('./histogram')
};

66
lib/cartodb/models/dataview/list.js
View File

@ -1,66 +0,0 @@
var dot = require('dot');
dot.templateSettings.strip = false;
var BaseWidget = require('./base');
var TYPE = 'list';
var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
/**
{
type: 'list',
options: {
columns: ['name', 'description']
}
}
*/
function List(query, options) {
options = options || {};
if (!Array.isArray(options.columns)) {
throw new Error('List expects `columns` array in widget options');
}
BaseWidget.apply(this);
this.query = query;
this.columns = options.columns;
}
List.prototype = new BaseWidget();
List.prototype.constructor = List;
module.exports = List;
List.prototype.sql = function(psql, override, callback) {
if (!callback) {
callback = override;
}
var listSql = listSqlTpl({
_query: this.query,
_columns: this.columns.join(', ')
});
return callback(null, listSql);
};
List.prototype.format = function(result) {
return {
rows: result.rows
};
};
List.prototype.getType = function() {
return TYPE;
};
List.prototype.toString = function() {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_columns: this.columns.join(', ')
});
};

3
lib/cartodb/models/dataview/overviews/index.js
View File

@ -1,6 +1,5 @@
module.exports = {
Aggregation: require('./aggregation'),
Formula: require('./formula'),
Histogram: require('./histogram'),
List: require('./list')
Histogram: require('./histogram')
};

11
lib/cartodb/models/dataview/overviews/list.js
View File

@ -1,11 +0,0 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../list');
function List(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
}
List.prototype = Object.create(BaseOverviewsDataview.prototype);
List.prototype.constructor = List;
module.exports = List;

22
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "3.12.11",
"version": "4.0.1",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@ -22,14 +22,14 @@
"Mario de Frutos <mario.defrutos@carto.com>"
],
"dependencies": {
"body-parser": "~1.14.0",
"camshaft": "0.58.1",
"cartodb-psql": "0.10.1",
"cartodb-query-tables": "0.2.0",
"body-parser": "^1.18.2",
"camshaft": "0.59.2",
"cartodb-psql": "0.10.2",
"cartodb-query-tables": "0.3.0",
"cartodb-redis": "0.14.0",
"debug": "~2.2.0",
"debug": "^3.1.0",
"dot": "~1.0.2",
"express": "~4.13.3",
"express": "~4.16.0",
"fastly-purge": "~1.0.1",
"log4js": "cartodb/log4js-node#cdb",
"lru-cache": "2.6.5",
@ -38,13 +38,13 @@
"on-headers": "^1.0.1",
"queue-async": "~1.0.7",
"redis-mpool": "0.4.1",
"request": "~2.79.0",
"request": "^2.83.0",
"semver": "~5.3.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"turbo-carto": "0.19.2",
"turbo-carto": "0.20.1",
"underscore": "~1.6.0",
"windshaft": "3.3.2",
"windshaft": "3.3.3",
"yargs": "~5.0.0"
},
"devDependencies": {
@ -63,6 +63,6 @@
},
"engines": {
"node": ">=6.9",
"yarn": "^0.21.3"
"yarn": ">=0.27.5 <1.0.0"
}
}

51
test/acceptance/widgets/list.js
View File

@ -1,51 +0,0 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('list widgets', function() {
it("should expose layer list", function(done) {
var listWidgetMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
names: {
type: 'list',
options: {
columns: ['name']
}
}
}
}
}
]
};
var testClient = new TestClient(listWidgetMapConfig);
testClient.getWidget('names', function(err, res) {
if (err) {
return done(err);
}
var expectedList = [
{name:"Hawai"},
{name:"El Estocolmo"},
{name:"El Rey del Tallarín"},
{name:"El Lacón"},
{name:"El Pico"}
];
assert.deepEqual(JSON.parse(res.body).rows, expectedList);
testClient.drain(done);
});
});
});

106
test/acceptance/widgets/ported/list.js
View File

@ -1,106 +0,0 @@
require('../../../support/test_helper');
var assert = require('../../../support/assert');
var TestClient = require('../../../support/test-client');
var _ = require('underscore');
describe('widgets', function() {
describe('lists', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
function listsMapConfig(columns) {
return {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table',
cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
cartocss_version: '2.0.1',
widgets: {
places: {
type: 'list',
options: {
columns: columns || ['name', 'address']
}
}
}
}
}
]
};
}
var EXPECTED_NAMES = ['Hawai', 'El Estocolmo', 'El Rey del Tallarín', 'El Lacón', 'El Pico'];
it('can be fetched from a valid list', function(done) {
var columns = ['name', 'address'];
this.testClient = new TestClient(listsMapConfig(columns));
this.testClient.getWidget('places', function (err, res, list) {
assert.ok(!err, err);
assert.ok(list);
assert.equal(list.type, 'list');
assert.equal(list.rows.length, 5);
assert.ok(onlyHasFields(list, columns));
var names = list.rows.map(function (item) {
return item.name;
});
assert.deepEqual(names, EXPECTED_NAMES);
var expectedAddresses = [
'Calle de Pérez Galdós 9, Madrid, Spain',
'Calle de la Palma 72, Madrid, Spain',
'Plaza Conde de Toreno 2, Madrid, Spain',
'Manuel Fernández y González 8, Madrid, Spain',
'Calle Divino Pastor 12, Madrid, Spain'
];
var addresses = list.rows.map(function (item) {
return item.address;
});
assert.deepEqual(addresses, expectedAddresses);
done();
});
});
it('should fetch just one column', function(done) {
var columns = ['name'];
this.testClient = new TestClient(listsMapConfig(columns));
this.testClient.getWidget('places', function (err, res, list) {
assert.ok(!err, err);
assert.ok(list);
assert.equal(list.type, 'list');
assert.equal(list.rows.length, 5);
assert.ok(onlyHasFields(list, columns));
var names = list.rows.map(function (item) {
return item.name;
});
assert.deepEqual(names, EXPECTED_NAMES);
done();
});
});
function onlyHasFields(list, expectedFields) {
var fields = (!!list.rows[0]) ? Object.keys(list.rows[0]) : [];
return _.difference(fields, expectedFields).length === 0 &&
_.difference(expectedFields, fields).length === 0;
}
});
});

109
test/support/assert.js
View File

@ -77,7 +77,6 @@ function randomImagePath() {
return path.resolve('test/results/png/image-test-' + Date.now() + '.png');
}
// jshint maxcomplexity:9
assert.response = function(server, req, res, callback) {
if (!callback) {
callback = res;
@ -106,7 +105,6 @@ assert.response = function(server, req, res, callback) {
// jshint maxcomplexity:9
function onServerListening() {
var status = res.status || res.statusCode;
var requestParams = {
url: 'http://' + host + ':' + port + req.url,
method: req.method || 'GET',
@ -122,61 +120,74 @@ assert.response = function(server, req, res, callback) {
request(requestParams, function assert$response$requestHandler(error, response, body) {
listener.close(function() {
response.body = response.body || body;
// Assert response body
if (res.body) {
var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
if (!eql) {
return callback(response, new Error(colorize(
'[red]{Invalid response body.}\n' +
' Expected: [green]{' + res.body + '}\n' +
' Got: [red]{' + response.body + '}'))
);
}
}
// Assert response status
if (typeof status === 'number') {
if (response.statusCode != status) {
return callback(response, new Error(colorize(
'[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body))
);
}
}
// Assert response headers
if (res.headers) {
var keys = Object.keys(res.headers);
for (var i = 0, len = keys.length; i < len; ++i) {
var name = keys[i],
actual = response.headers[name.toLowerCase()],
expected = res.headers[name],
headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
if (!headerEql) {
return callback(response, new Error(colorize(
'Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expected + '}\n' +
' Got: [red]{' + actual + '}'))
);
}
}
}
// Callback
callback(response);
var err = validateResponse(response, res);
return callback(response, err);
});
});
}
};
// jshint maxcomplexity:6
function validateResponseBody(response, expected) {
if (expected.body) {
var eql = expected.body instanceof RegExp ?
expected.body.test(response.body) :
expected.body === response.body;
if (!eql) {
return new Error(colorize(
'[red]{Invalid response body.}\n' +
' Expected: [green]{' + expected.body + '}\n' +
' Got: [red]{' + response.body + '}')
);
}
}
}
function validateResponseStatus(response, expected) {
var status = expected.status || expected.statusCode;
// Assert response status
if (typeof status === 'number') {
if (response.statusCode !== status) {
return new Error(colorize(
'[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body)
);
}
}
}
function validateResponseHeaders(response, expected) {
// Assert response headers
if (expected.headers) {
var keys = Object.keys(expected.headers);
for (var i = 0, len = keys.length; i < len; ++i) {
var name = keys[i],
actual = response.headers[name.toLowerCase()],
expectedHeader = expected.headers[name],
headerEql = expectedHeader instanceof RegExp ? expectedHeader.test(actual) : expectedHeader === actual;
if (!headerEql) {
return new Error(colorize(
'Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expectedHeader + '}\n' +
' Got: [red]{' + actual + '}')
);
}
}
}
}
function validateResponse(response, expected) {
// Assert response body
return validateResponseBody(response, expected) ||
validateResponseStatus(response, expected) ||
validateResponseHeaders(response, expected);
}
// @param tolerance number of tolerated grid cell differences
// jshint maxcomplexity:9
assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
// jshint maxcomplexity:9
fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));

21
test/support/config.js
View File

@ -1,21 +0,0 @@
var _ = require('underscore');
require(__dirname + '/test_helper');
module.exports = function(opts) {
var config = {
redis_pool: {
max: 10,
idleTimeoutMillis: 1,
reapIntervalMillis: 1,
port: global.environment.redis.port
}
}
_.extend(config, opts || {});
return config;
}();

223
test/support/test-client.js
View File

@ -22,7 +22,7 @@ const MAPNIK_SUPPORTED_FORMATS = {
'grid.json': true,
'geojson': true,
'mvt': true
}
};
function TestClient(config, apiKey) {
this.mapConfig = isMapConfig(config) ? config : null;
@ -115,6 +115,15 @@ module.exports.CARTOCSS = {
module.exports.SQL = {
EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator',
ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator'
};
function resErr2errRes(callback) {
return (res, err) => {
if (err) {
return callback(err);
}
return callback(err, res);
};
}
TestClient.prototype.getWidget = function(widgetName, params, callback) {
@ -130,7 +139,6 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
}
var layergroupId;
step(
function createLayergroup() {
var next = this;
@ -176,11 +184,12 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
}
);
},
function getWidgetResult(err, _layergroupId) {
function getWidgetResult(err, layergroupId) {
assert.ifError(err);
var next = this;
layergroupId = _layergroupId;
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
var urlParams = {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
@ -217,8 +226,6 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
);
},
function finish(err, res) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
var widget;
if (!err && res.body) {
widget = JSON.parse(res.body);
@ -241,7 +248,6 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
}
var layergroupId;
step(
function createLayergroup() {
var next = this;
@ -287,11 +293,12 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
}
);
},
function getWidgetSearchResult(err, _layergroupId) {
function getWidgetSearchResult(err, layergroupId) {
assert.ifError(err);
var next = this;
layergroupId = _layergroupId;
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
var urlParams = {
q: userQuery,
@ -326,8 +333,6 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
);
},
function finish(err, res) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
var searchResult;
if (!err && res.body) {
searchResult = JSON.parse(res.body);
@ -365,7 +370,6 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
}
};
var layergroupId;
step(
function createLayergroup() {
var next = this;
@ -401,11 +405,12 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
}
);
},
function getDataviewResult(err, _layergroupId) {
function getDataviewResult(err, layergroupId) {
assert.ifError(err);
var next = this;
layergroupId = _layergroupId;
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
var urlParams = {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
@ -444,12 +449,6 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
if (err) {
return callback(err);
}
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(null, dataview);
}
);
@ -483,7 +482,6 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params,
}
};
var layergroupId;
step(
function createLayergroup() {
var next = this;
@ -572,7 +570,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
var layergroupId;
if (params.layergroupid) {
layergroupId = params.layergroupid
layergroupId = params.layergroupid;
}
step(
@ -587,7 +585,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
return next(new Error('apiKey param is mandatory to create a new template'));
}
params.placeholders = params.placeholders || {};
params.placeholders = params.placeholders || {};
assert.response(self.server,
{
@ -620,7 +618,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
return next(null, layergroupId);
}
var data = templateId ? params.placeholders : self.mapConfig
var data = templateId ? params.placeholders : self.mapConfig;
var path = templateId ?
urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) :
url;
@ -649,11 +647,12 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
);
},
function getTileResult(err, _layergroupId) {
function getTileResult(err, layergroupId) {
// jshint maxcomplexity:12
assert.ifError(err);
var next = this;
layergroupId = _layergroupId;
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
url = '/api/v1/map/' + layergroupId + '/';
@ -727,35 +726,31 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/x-protobuf':
body = new mapnik.VectorTile(z, x, y);
body.setDataSync(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, body);
});
assert.response(self.server, request, expectedResponse, resErr2errRes(this));
},
function finish(err, res, image) {
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
function finish(err, res) {
if (err) {
return callback(err);
}
return callback(err, res, image);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/x-protobuf':
body = new mapnik.VectorTile(z, x, y);
body.setDataSync(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body;
break;
}
return callback(err, res, body);
}
);
};
@ -791,10 +786,11 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
},
expectedResponse,
function(res, err) {
var parsedBody;
// If there is a response, we are still interested in catching the created keys
// to be able to delete them on the .drain() method.
if (res) {
var parsedBody = JSON.parse(res.body);
parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
@ -812,7 +808,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
TestClient.prototype.getStaticCenter = function (params, callback) {
var self = this;
let { layergroupid, z, lat, lng, width, height, format } = params
let { layergroupid, z, lat, lng, width, height, format } = params;
var url = `/api/v1/map/`;
@ -828,7 +824,7 @@ TestClient.prototype.getStaticCenter = function (params, callback) {
return next(null, layergroupid);
}
var data = self.mapConfig
var data = self.mapConfig;
var path = url;
assert.response(self.server,
@ -855,14 +851,13 @@ TestClient.prototype.getStaticCenter = function (params, callback) {
}
);
},
function getStaticResult(err, _layergroupid) {
function getStaticResult(err, layergroupId) {
assert.ifError(err);
var next = this;
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
layergroupid = _layergroupid;
url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}`
url = `/api/v1/map/static/center/${layergroupId}/${z}/${lat}/${lng}/${width}/${height}.${format}`;
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
@ -884,31 +879,27 @@ TestClient.prototype.getStaticCenter = function (params, callback) {
}
}, params.response);
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, body);
});
assert.response(self.server, request, expectedResponse, resErr2errRes(this));
},
function finish(err, res, image) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
function(err, res) {
if (err) {
return callback(err);
}
return callback(err, res, image);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body;
break;
}
return callback(err, res, body);
}
);
};
@ -922,7 +913,6 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
var layergroupId;
var nodes = {};
step(
function createLayergroup() {
@ -961,11 +951,11 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
}
);
},
function getNodeStatusResult(err, _layergroupId) {
function getNodeStatusResult(err, layergroupId) {
assert.ifError(err);
var next = this;
layergroupId = _layergroupId;
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
url = urlParser.parse(nodes[nodeName]).path;
@ -988,15 +978,13 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
}
};
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
next(null, res, JSON.parse(res.body));
});
assert.response(self.server, request, expectedResponse, resErr2errRes(this));
},
function finish(err, res, image) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
return callback(err, res, image);
function finish(err, res) {
if (err) {
return callback(err);
}
return callback(null, res, JSON.parse(res.body));
}
);
};
@ -1005,11 +993,11 @@ TestClient.prototype.getAttributes = function(params, callback) {
var self = this;
if (!Number.isFinite(params.featureId)) {
throw new Error('featureId param must be a number')
throw new Error('featureId param must be a number');
}
if (!Number.isFinite(params.layer)) {
throw new Error('layer param must be a number')
throw new Error('layer param must be a number');
}
var url = '/api/v1/map';
@ -1021,7 +1009,7 @@ TestClient.prototype.getAttributes = function(params, callback) {
var layergroupid;
if (params.layergroupid) {
layergroupid = params.layergroupid
layergroupid = params.layergroupid;
}
step(
@ -1058,14 +1046,13 @@ TestClient.prototype.getAttributes = function(params, callback) {
}
);
},
function getAttributes(err, _layergroupid) {
function getAttributes(err, layergroupId) {
assert.ifError(err);
var next = this;
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
layergroupid = _layergroupid;
url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`;
url = `/api/v1/map/${layergroupId}/${params.layer}/attributes/${params.featureId}`;
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
@ -1086,21 +1073,14 @@ TestClient.prototype.getAttributes = function(params, callback) {
}
};
assert.response(self.server, request, expectedResponse, function (res, err) {
assert.ifError(err);
var attributes = JSON.parse(res.body);
next(null, res, attributes);
});
assert.response(self.server, request, expectedResponse, resErr2errRes(this));
},
function finish(err, res, attributes) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
function finish(err, res) {
if (err) {
return callback(err);
}
return callback(err, res, attributes);
var attributes = JSON.parse(res.body);
return callback(null, res, attributes);
}
);
};
@ -1144,7 +1124,7 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
assert.response(self.server, requestOptions, expectedResponse, function (res, err) {
assert.response(server, requestOptions, expectedResponse, function (res, err) {
helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() {
return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary')));
});
@ -1166,12 +1146,11 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi
TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) {
const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db';
const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 })
const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 })
const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 });
const publicuser = global.environment.postgres.user;
// we need to guarantee all new connections have the new settings
helper.cleanPGPoolConnections()
helper.cleanPGPoolConnections();
const psql = new PSQL({
user: 'postgres',

73
test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js
View File

@ -143,79 +143,6 @@ describe('dataviews-widgets-adapter', function() {
}
}
},
{
"input": {
"version": "1.4.0",
"layers": [
{
"type": "mapnik",
"options": {
"sql": "select * from test_table",
"cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
"cartocss_version": "2.3.0",
"widgets": {
"names": {
"type": "list",
"options": {
"columns": [
"name"
]
}
}
}
}
}
]
},
"expected": {
"version": "1.4.0",
"layers": [
{
"type": "mapnik",
"options": {
"source": {
"id": "cdb-layer-source-0"
},
"cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
"cartocss_version": "2.3.0",
// keep them for now
"widgets": {
"names": {
"type": "list",
"options": {
"columns": [
"name"
]
}
}
}
}
}
],
"analyses": [
{
"id": "cdb-layer-source-0",
"type": "source",
"params": {
"query": "select * from test_table"
}
}
],
"dataviews": {
"names": {
"source": {
"id": "cdb-layer-source-0"
},
"type": "list",
"options": {
"columns": [
"name"
]
}
}
}
}
},
{
"input": {
"version": "1.4.0",

487
yarn.lock
View File

@ -10,16 +10,20 @@ abaculus@cartodb/abaculus#2.0.3-cdb1:
mapnik "~3.5.0"
sphericalmercator "1.0.x"
abbrev@1, abbrev@1.0.x:
abbrev@1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
abbrev@1.0.x:
version "1.0.9"
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135"
accepts@~1.2.12:
version "1.2.13"
resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.2.13.tgz#e5f1f3928c6d95fd96558c36ec3d9d0de4a6ecea"
accepts@~1.3.4:
version "1.3.4"
resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f"
dependencies:
mime-types "~2.1.6"
negotiator "0.5.3"
mime-types "~2.1.16"
negotiator "0.6.1"
ajv@^4.9.1:
version "4.11.8"
@ -28,6 +32,15 @@ ajv@^4.9.1:
co "^4.6.0"
json-stable-stringify "^1.0.1"
ajv@^5.1.0:
version "5.2.3"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.3.tgz#c06f598778c44c6b161abafe3466b81ad1814ed2"
dependencies:
co "^4.6.0"
fast-deep-equal "^1.0.0"
json-schema-traverse "^0.3.0"
json-stable-stringify "^1.0.1"
align-text@^0.1.1, align-text@^0.1.3:
version "0.1.4"
resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
@ -125,20 +138,20 @@ block-stream@*:
dependencies:
inherits "~2.0.0"
body-parser@~1.14.0:
version "1.14.2"
resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.14.2.tgz#1015cb1fe2c443858259581db53332f8d0cf50f9"
body-parser@1.18.2, body-parser@^1.18.2:
version "1.18.2"
resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.2.tgz#87678a19d84b47d859b83199bd59bce222b10454"
dependencies:
bytes "2.2.0"
content-type "~1.0.1"
debug "~2.2.0"
depd "~1.1.0"
http-errors "~1.3.1"
iconv-lite "0.4.13"
bytes "3.0.0"
content-type "~1.0.4"
debug "2.6.9"
depd "~1.1.1"
http-errors "~1.6.2"
iconv-lite "0.4.19"
on-finished "~2.3.0"
qs "5.2.0"
raw-body "~2.1.5"
type-is "~1.6.10"
qs "6.5.1"
raw-body "2.3.2"
type-is "~1.6.15"
boom@2.x.x:
version "2.10.1"
@ -174,13 +187,9 @@ bunyan@1.8.1:
mv "~2"
safe-json-stringify "~1"
bytes@2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-2.2.0.tgz#fd35464a403f6f9117c2de3609ecff9cae000588"
bytes@2.4.0:
version "2.4.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-2.4.0.tgz#7d97196f9d5baf7f6935e25985549edd2a6c2339"
bytes@3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"
camelcase@^1.0.2:
version "1.2.1"
@ -190,14 +199,14 @@ camelcase@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
camshaft@0.58.1:
version "0.58.1"
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.1.tgz#e4156580683f624212ea3020e59790ad006f24cc"
camshaft@0.59.2:
version "0.59.2"
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.2.tgz#8b032771faa1264bd8a81040c6075beb1a32e286"
dependencies:
async "^1.5.2"
bunyan "1.8.1"
cartodb-psql "^0.10.1"
debug "^2.2.0"
debug "^3.1.0"
dot "^1.0.3"
request "^2.69.0"
@ -241,17 +250,17 @@ cartocolor@4.0.0:
dependencies:
colorbrewer "1.0.0"
cartodb-psql@0.10.1, cartodb-psql@^0.10.1:
version "0.10.1"
resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.1.tgz#0ac947e62fe10b27916df6b7ba6c461953fe3a23"
cartodb-psql@0.10.2, cartodb-psql@^0.10.1:
version "0.10.2"
resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.2.tgz#8c505066e4a635cfa0ee4c603769c83f6e2187dd"
dependencies:
debug "~2.2.0"
debug "^3.1.0"
pg cartodb/node-postgres#6.1.6-cdb1
underscore "~1.6.0"
cartodb-query-tables@0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44"
cartodb-query-tables@0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756"
cartodb-redis@0.14.0:
version "0.14.0"
@ -261,10 +270,6 @@ cartodb-redis@0.14.0:
redis-mpool "~0.4.1"
underscore "~1.6.0"
caseless@~0.11.0:
version "0.11.0"
resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.11.0.tgz#715b96ea9841593cc33067923f5ec60ebda4f7d7"
caseless@~0.12.0:
version "0.12.0"
resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@ -284,7 +289,7 @@ center-align@^0.1.1:
deep-eql "^0.1.3"
type-detect "^1.0.0"
chalk@^1.1.1, chalk@^1.1.3:
chalk@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
dependencies:
@ -359,11 +364,11 @@ console-control-strings@^1.0.0, console-control-strings@~1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
content-disposition@0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.1.tgz#87476c6a67c8daa87e32e87616df883ba7fb071b"
content-disposition@0.5.2:
version "0.5.2"
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4"
content-type@~1.0.1:
content-type@~1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
@ -371,9 +376,9 @@ cookie-signature@1.0.6:
version "1.0.6"
resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
cookie@0.1.5:
version "0.1.5"
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.1.5.tgz#6ab9948a4b1ae21952cd2588530a4722d4044d7c"
cookie@0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.3.1.tgz#e7e0a1f9ef43b4c8ba925c5c5a96e806d16873bb"
core-util-is@1.0.2, core-util-is@~1.0.0:
version "1.0.2"
@ -399,24 +404,30 @@ date-now@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
debug@2.2.0, debug@^2.2.0, debug@~2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da"
dependencies:
ms "0.7.1"
debug@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b"
dependencies:
ms "0.7.2"
debug@2.6.9, debug@^2.2.0:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
dependencies:
ms "2.0.0"
debug@^1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac"
dependencies:
ms "2.0.0"
debug@^3.1.0, debug@~3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
dependencies:
ms "2.0.0"
decamelize@^1.0.0, decamelize@^1.1.1:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@ -447,7 +458,7 @@ delegates@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
depd@~1.1.0:
depd@1.1.1, depd@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359"
@ -507,6 +518,10 @@ ee-first@1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
encodeurl@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.1.tgz#79e3d58655346909fe6f0f45a5de68103b294d20"
entities@1.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/entities/-/entities-1.0.0.tgz#b2987aa3821347fcde642b24fdfc9e4fb712bf26"
@ -560,43 +575,48 @@ esutils@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b"
etag@~1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/etag/-/etag-1.7.0.tgz#03d30b5f67dd6e632d2945d30d6652731a34d5d8"
etag@~1.8.1:
version "1.8.1"
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
exit@0.1.2, exit@0.1.x:
version "0.1.2"
resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c"
express@~4.13.3:
version "4.13.4"
resolved "https://registry.yarnpkg.com/express/-/express-4.13.4.tgz#3c0b76f3c77590c8345739061ec0bd3ba067ec24"
express@~4.16.0:
version "4.16.1"
resolved "https://registry.yarnpkg.com/express/-/express-4.16.1.tgz#6b33b560183c9b253b7b62144df33a4654ac9ed0"
dependencies:
accepts "~1.2.12"
accepts "~1.3.4"
array-flatten "1.1.1"
content-disposition "0.5.1"
content-type "~1.0.1"
cookie "0.1.5"
body-parser "1.18.2"
content-disposition "0.5.2"
content-type "~1.0.4"
cookie "0.3.1"
cookie-signature "1.0.6"
debug "~2.2.0"
depd "~1.1.0"
debug "2.6.9"
depd "~1.1.1"
encodeurl "~1.0.1"
escape-html "~1.0.3"
etag "~1.7.0"
finalhandler "0.4.1"
fresh "0.3.0"
etag "~1.8.1"
finalhandler "1.1.0"
fresh "0.5.2"
merge-descriptors "1.0.1"
methods "~1.1.2"
on-finished "~2.3.0"
parseurl "~1.3.1"
parseurl "~1.3.2"
path-to-regexp "0.1.7"
proxy-addr "~1.0.10"
qs "4.0.0"
range-parser "~1.0.3"
send "0.13.1"
serve-static "~1.10.2"
type-is "~1.6.6"
utils-merge "1.0.0"
vary "~1.0.1"
proxy-addr "~2.0.2"
qs "6.5.1"
range-parser "~1.2.0"
safe-buffer "5.1.1"
send "0.16.1"
serve-static "1.13.1"
setprototypeof "1.1.0"
statuses "~1.3.1"
type-is "~1.6.15"
utils-merge "1.0.1"
vary "~1.1.2"
extend@~3.0.0:
version "3.0.1"
@ -616,13 +636,16 @@ fastly-purge@~1.0.1:
dependencies:
request "^2.55.0"
finalhandler@0.4.1:
version "0.4.1"
resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-0.4.1.tgz#85a17c6c59a94717d262d61230d4b0ebe3d4a14d"
finalhandler@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.0.tgz#ce0b6855b45853e791b2fcc680046d88253dd7f5"
dependencies:
debug "~2.2.0"
debug "2.6.9"
encodeurl "~1.0.1"
escape-html "~1.0.3"
on-finished "~2.3.0"
parseurl "~1.3.2"
statuses "~1.3.1"
unpipe "~1.0.0"
find-up@^1.0.0:
@ -644,13 +667,21 @@ form-data@~2.1.1:
combined-stream "^1.0.5"
mime-types "^2.1.12"
forwarded@~0.1.0:
form-data@~2.3.1:
version "2.3.1"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf"
dependencies:
asynckit "^0.4.0"
combined-stream "^1.0.5"
mime-types "^2.1.12"
forwarded@~0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"
fresh@0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.3.0.tgz#651f838e22424e7566de161d8358caa199f83d4f"
fresh@0.5.2:
version "0.5.2"
resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7"
fs.realpath@^1.0.0:
version "1.0.0"
@ -693,16 +724,6 @@ gdal@~0.9.2:
nan "~2.6.2"
node-pre-gyp "~0.6.36"
generate-function@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/generate-function/-/generate-function-2.0.0.tgz#6858fe7c0969b7d4e9093337647ac79f60dfbe74"
generate-object-property@^1.1.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/generate-object-property/-/generate-object-property-1.2.0.tgz#9c0e1c40308ce804f4783618b937fa88f99d50d0"
dependencies:
is-property "^1.0.0"
generic-pool@2.4.3:
version "2.4.3"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff"
@ -769,11 +790,11 @@ graceful-fs@^4.1.2:
resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725"
grainstore@~1.6.0:
version "1.6.3"
resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.3.tgz#6900cc811aadc1ed2c00fcd429c672f8b8e1a5cb"
version "1.6.4"
resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.4.tgz#617b93c5e2de8f544375202da89b9208a8b3d762"
dependencies:
carto "0.16.3"
debug "~2.2.0"
debug "~3.1.0"
generic-pool "~2.2.0"
millstone "0.6.17"
postcss "~5.2.8"
@ -800,14 +821,9 @@ har-schema@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e"
har-validator@~2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d"
dependencies:
chalk "^1.1.1"
commander "^2.9.0"
is-my-json-valid "^2.12.4"
pinkie-promise "^2.0.0"
har-schema@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
har-validator@~4.2.1:
version "4.2.1"
@ -864,12 +880,14 @@ htmlparser2@3.8.x:
entities "1.0"
readable-stream "1.1"
http-errors@~1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.3.1.tgz#197e22cdebd4198585e8694ef6786197b91ed942"
http-errors@1.6.2, http-errors@~1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.2.tgz#0a002cc85707192a7e7946ceedc11155f60ec736"
dependencies:
inherits "~2.0.1"
statuses "1"
depd "1.1.1"
inherits "2.0.3"
setprototypeof "1.0.3"
statuses ">= 1.3.1 < 2"
http-signature@~1.1.0:
version "1.1.1"
@ -883,9 +901,9 @@ husl@^6.0.1:
version "6.0.6"
resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d"
iconv-lite@0.4.13:
version "0.4.13"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.13.tgz#1f88aba4ab0b1508e8312acc39345f36e992e2f2"
iconv-lite@0.4.19:
version "0.4.19"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b"
inflight@^1.0.4:
version "1.0.6"
@ -894,7 +912,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
inherits@2, inherits@2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
@ -906,9 +924,9 @@ invert-kv@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6"
ipaddr.js@1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.0.5.tgz#5fa78cf301b825c78abc3042d812723049ea23c7"
ipaddr.js@1.5.2:
version "1.5.2"
resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.5.2.tgz#d4b505bde9946987ccf0fc58d9010ff9607e3fa0"
is-arrayish@^0.2.1:
version "0.2.1"
@ -930,19 +948,6 @@ is-fullwidth-code-point@^1.0.0:
dependencies:
number-is-nan "^1.0.0"
is-my-json-valid@^2.12.4:
version "2.16.1"
resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11"
dependencies:
generate-function "^2.0.0"
generate-object-property "^1.1.0"
jsonpointer "^4.0.0"
xtend "^4.0.0"
is-property@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84"
is-typedarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
@ -1040,10 +1045,6 @@ jsonify@~0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
jsonpointer@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-4.0.1.tgz#4fd92cb34e0e9db3c89c8622ecf51f9b978c6cb9"
jsprim@^1.2.2:
version "1.4.1"
resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
@ -1231,15 +1232,15 @@ mime-db@~1.30.0:
version "1.30.0"
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01"
mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7:
mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17, mime-types@~2.1.7:
version "2.1.17"
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a"
dependencies:
mime-db "~1.30.0"
mime@1.3.4, mime@~1.3.4:
version "1.3.4"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53"
mime@1.4.1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6"
mime@~1.2.11:
version "1.2.11"
@ -1289,10 +1290,6 @@ moment@^2.10.6, moment@~2.18.1:
version "2.18.1"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"
ms@0.7.1:
version "0.7.1"
resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098"
ms@0.7.2:
version "0.7.2"
resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765"
@ -1325,9 +1322,9 @@ ncp@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3"
negotiator@0.5.3:
version "0.5.3"
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.5.3.tgz#269d5c476810ec92edbe7b6c2f28316384f9a7e8"
negotiator@0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9"
nock@~2.11.0:
version "2.11.0"
@ -1475,7 +1472,7 @@ parse-json@^2.2.0:
dependencies:
error-ex "^1.2.0"
parseurl@~1.3.1:
parseurl@~1.3.2:
version "1.3.2"
resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3"
@ -1633,28 +1630,20 @@ protozero@~1.4.2:
version "1.4.5"
resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.4.5.tgz#80eaa80a4f9c751465c4cb2620d8233b50ec1aff"
proxy-addr@~1.0.10:
version "1.0.10"
resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-1.0.10.tgz#0d40a82f801fc355567d2ecb65efe3f077f121c5"
proxy-addr@~2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.2.tgz#6571504f47bb988ec8180253f85dd7e14952bdec"
dependencies:
forwarded "~0.1.0"
ipaddr.js "1.0.5"
forwarded "~0.1.2"
ipaddr.js "1.5.2"
punycode@^1.4.1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
qs@4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/qs/-/qs-4.0.0.tgz#c31d9b74ec27df75e543a86c78728ed8d4623607"
qs@5.2.0:
version "5.2.0"
resolved "https://registry.yarnpkg.com/qs/-/qs-5.2.0.tgz#a9f31142af468cb72b25b30136ba2456834916be"
qs@~6.3.0:
version "6.3.2"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.3.2.tgz#e75bd5f6e268122a2a0e0bda630b2550c166502c"
qs@6.5.1, qs@~6.5.1:
version "6.5.1"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8"
qs@~6.4.0:
version "6.4.0"
@ -1664,16 +1653,17 @@ queue-async@~1.0.7:
version "1.0.7"
resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945"
range-parser@~1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.0.3.tgz#6872823535c692e2c2a0103826afd82c2e0ff175"
range-parser@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e"
raw-body@~2.1.5:
version "2.1.7"
resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.1.7.tgz#adfeace2e4fb3098058014d08c072dcc59758774"
raw-body@2.3.2:
version "2.3.2"
resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.2.tgz#bcd60c77d3eb93cde0050295c3f379389bc88f89"
dependencies:
bytes "2.4.0"
iconv-lite "0.4.13"
bytes "3.0.0"
http-errors "1.6.2"
iconv-lite "0.4.19"
unpipe "1.0.0"
rc@^1.1.7:
@ -1774,30 +1764,32 @@ request@2.81.0:
tunnel-agent "^0.6.0"
uuid "^3.0.0"
request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0:
version "2.79.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
request@2.x, request@^2.55.0, request@^2.69.0, request@^2.83.0:
version "2.83.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356"
dependencies:
aws-sign2 "~0.6.0"
aws4 "^1.2.1"
caseless "~0.11.0"
aws-sign2 "~0.7.0"
aws4 "^1.6.0"
caseless "~0.12.0"
combined-stream "~1.0.5"
extend "~3.0.0"
extend "~3.0.1"
forever-agent "~0.6.1"
form-data "~2.1.1"
har-validator "~2.0.6"
hawk "~3.1.3"
http-signature "~1.1.0"
form-data "~2.3.1"
har-validator "~5.0.3"
hawk "~6.0.2"
http-signature "~1.2.0"
is-typedarray "~1.0.0"
isstream "~0.1.2"
json-stringify-safe "~5.0.1"
mime-types "~2.1.7"
oauth-sign "~0.8.1"
qs "~6.3.0"
stringstream "~0.0.4"
tough-cookie "~2.3.0"
tunnel-agent "~0.4.1"
uuid "^3.0.0"
mime-types "~2.1.17"
oauth-sign "~0.8.2"
performance-now "^2.1.0"
qs "~6.5.1"
safe-buffer "^5.1.1"
stringstream "~0.0.5"
tough-cookie "~2.3.3"
tunnel-agent "^0.6.0"
uuid "^3.1.0"
require-directory@^2.1.1:
version "2.1.1"
@ -1829,7 +1821,7 @@ rimraf@~2.4.0:
dependencies:
glob "^6.0.1"
safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
@ -1853,52 +1845,49 @@ semver@~5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a"
send@0.13.1:
version "0.13.1"
resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7"
dependencies:
debug "~2.2.0"
depd "~1.1.0"
destroy "~1.0.4"
escape-html "~1.0.3"
etag "~1.7.0"
fresh "0.3.0"
http-errors "~1.3.1"
mime "1.3.4"
ms "0.7.1"
on-finished "~2.3.0"
range-parser "~1.0.3"
statuses "~1.2.1"
semver@~5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
send@0.13.2:
version "0.13.2"
resolved "https://registry.yarnpkg.com/send/-/send-0.13.2.tgz#765e7607c8055452bba6f0b052595350986036de"
send@0.16.1:
version "0.16.1"
resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3"
dependencies:
debug "~2.2.0"
depd "~1.1.0"
debug "2.6.9"
depd "~1.1.1"
destroy "~1.0.4"
encodeurl "~1.0.1"
escape-html "~1.0.3"
etag "~1.7.0"
fresh "0.3.0"
http-errors "~1.3.1"
mime "1.3.4"
ms "0.7.1"
etag "~1.8.1"
fresh "0.5.2"
http-errors "~1.6.2"
mime "1.4.1"
ms "2.0.0"
on-finished "~2.3.0"
range-parser "~1.0.3"
statuses "~1.2.1"
range-parser "~1.2.0"
statuses "~1.3.1"
serve-static@~1.10.2:
version "1.10.3"
resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.10.3.tgz#ce5a6ecd3101fed5ec09827dac22a9c29bfb0535"
serve-static@1.13.1:
version "1.13.1"
resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.1.tgz#4c57d53404a761d8f2e7c1e8a18a47dbf278a719"
dependencies:
encodeurl "~1.0.1"
escape-html "~1.0.3"
parseurl "~1.3.1"
send "0.13.2"
parseurl "~1.3.2"
send "0.16.1"
set-blocking@^2.0.0, set-blocking@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
setprototypeof@1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.0.3.tgz#66567e37043eeb4f04d91bd658c0cbefb55b8e04"
setprototypeof@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656"
shelljs@0.3.x:
version "0.3.0"
resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1"
@ -1966,8 +1955,8 @@ sprintf-js@~1.0.2:
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
"sqlite3@2.x || 3.x":
version "3.1.12"
resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.12.tgz#2b3a14b17162e39e8aa6e1e2487a41d0795396d8"
version "3.1.13"
resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.13.tgz#d990a05627392768de6278bafd1a31fdfe907dd9"
dependencies:
nan "~2.7.0"
node-pre-gyp "~0.6.38"
@ -1992,14 +1981,10 @@ sshpk@^1.7.0:
jsbn "~0.1.0"
tweetnacl "~0.14.0"
statuses@1:
"statuses@>= 1.3.1 < 2", statuses@~1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e"
statuses@~1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.2.1.tgz#dded45cc18256d51ed40aec142489d5c61026d28"
step-profiler@~0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/step-profiler/-/step-profiler-0.3.0.tgz#841368ce44f2330c862edd5ec60e7fbec148f0b3"
@ -2137,7 +2122,7 @@ torque.js@~2.11.0:
dependencies:
carto CartoDB/carto#0.15.1-cdb1
tough-cookie@~2.3.0:
tough-cookie@~2.3.0, tough-cookie@~2.3.3:
version "2.3.3"
resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561"
dependencies:
@ -2149,17 +2134,13 @@ tunnel-agent@^0.6.0:
dependencies:
safe-buffer "^5.0.1"
tunnel-agent@~0.4.1:
version "0.4.3"
resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb"
turbo-carto@0.19.2:
version "0.19.2"
resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd"
turbo-carto@0.20.1:
version "0.20.1"
resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.1.tgz#e9f5fa1408d9d4325a1e79333e6d242170f89e6d"
dependencies:
cartocolor "4.0.0"
colorbrewer "1.0.0"
debug "2.2.0"
debug "^3.1.0"
es6-promise "3.1.2"
postcss "5.0.19"
postcss-value-parser "3.3.0"
@ -2182,7 +2163,7 @@ type-detect@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2"
type-is@~1.6.10, type-is@~1.6.6:
type-is@~1.6.15:
version "1.6.15"
resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410"
dependencies:
@ -2226,9 +2207,9 @@ util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
utils-merge@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8"
utils-merge@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
uuid@^3.0.0:
version "3.1.0"
@ -2241,9 +2222,9 @@ validate-npm-package-license@^3.0.1:
spdx-correct "~1.0.0"
spdx-expression-parse "~1.0.0"
vary@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/vary/-/vary-1.0.1.tgz#99e4981566a286118dfb2b817357df7993376d10"
vary@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
verror@1.10.0:
version "1.10.0"
@ -2277,21 +2258,21 @@ window-size@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@3.3.2:
version "3.3.2"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.2.tgz#72efe0dbc0d8d4bcba4211fdabd15dd2e0799df9"
windshaft@3.3.3:
version "3.3.3"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.3.tgz#0582e6a0d9cf91c533134787ace64a3337200e33"
dependencies:
abaculus cartodb/abaculus#2.0.3-cdb1
canvas cartodb/node-canvas#1.6.2-cdb2
carto cartodb/carto#0.15.1-cdb3
cartodb-psql "^0.10.1"
debug "~2.2.0"
debug "^3.1.0"
dot "~1.0.2"
grainstore "~1.6.0"
mapnik "3.5.14"
queue-async "~1.0.7"
redis-mpool "0.4.1"
request "~2.79.0"
request "^2.83.0"
semver "~5.0.3"
sphericalmercator "1.0.4"
step "~0.0.6"