cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into errorLogs

Browse Source
This commit is contained in:
Simon Martín 2017-12-11 12:59:42 +01:00
commit 1c6c3962db
12 changed files with 676 additions and 412 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
NEWS.md
View File

@ -1,5 +1,36 @@
# Changelog # Changelog
## 4.3.1
Released 2017-mm-dd
Announcements:
## 4.3.0
Released 2017-12-11
Announcements:
- Optimize Formula queries.
- Optimize Formula queries in overviews.
- Optimize Numeric Histogram queries.
- Optimize Date Histogram queries.
- Date Histograms: Now returns the same value for max/min/avg/timestamp per bin.
- Date Histograms: Now it should return the same no matter the DB/Client time zone.
## 4.2.0
Released 2017-12-04
Announcements:
- Allow to request MVT tiles without CartoCSS
- Upgrades windshaft to [4.1.0](https://github.com/CartoDB/windshaft/releases/tag/4.1.0).
## 4.1.1
Released 2017-11-29
Announcements:
- Upgrades turbo-carto to [0.20.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.2).
## 4.1.0 ## 4.1.0
Released 2017-mm-dd Released 2017-mm-dd

36
lib/cartodb/models/dataview/formula.js
View File

@ -1,34 +1,14 @@
const BaseDataview = require('./base'); const BaseDataview = require('./base');
const debug = require('debug')('windshaft:dataview:formula'); const debug = require('debug')('windshaft:dataview:formula');
const utils = require('../../utils/query-utils');
const countInfinitiesQueryTpl = ctx => ` const formulaQueryTpl = ctx =>
SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities `SELECT
WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float ${ctx.operation}(${utils.handleFloatColumn(ctx)}) AS result,
`; ${utils.countNULLs(ctx)} AS nulls_count
${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
const countNansQueryTpl = ctx => ` ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans FROM (${ctx.query}) __cdb_formula`;
WHERE ${ctx.column} = 'NaN'::float
`;
const filterOutSpecialNumericValuesTpl = ctx => `
WHERE
${ctx.column} != 'infinity'::float
AND
${ctx.column} != '-infinity'::float
AND
${ctx.column} != 'NaN'::float
`;
const formulaQueryTpl = ctx => `
SELECT
${ctx.operation}(${ctx.column}) AS result,
(SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count
${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''}
${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''}
FROM (${ctx.query}) __cdb_formula
${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''}
`;
const VALID_OPERATIONS = { const VALID_OPERATIONS = {
count: true, count: true,

233
lib/cartodb/models/dataview/histograms/date-histogram.js
View File

@ -1,5 +1,102 @@
const BaseHistogram = require('./base-histogram'); const BaseHistogram = require('./base-histogram');
const debug = require('debug')('windshaft:dataview:date-histogram'); const debug = require('debug')('windshaft:dataview:date-histogram');
const utils = require('../../../utils/query-utils');
/**
* Gets the name of a timezone with the same offset as the required
* using the pg_timezone_names table. We do this because it's simpler to pass
* the name than to pass the offset itself as PostgreSQL uses different
* sign convention. For example: TIME ZONE 'CET' is equal to TIME ZONE 'UTC-1',
* not 'UTC+1' which would be expected.
* Gives priority to Etc/GMT±N timezones but still support odd offsets like 8.5
* hours for Asia/Pyongyang.
* It also makes it easier to, in the future, support the input of expected timezone
* instead of the offset; that is using 'Europe/Madrid' instead of
* '+3600' or '+7200'. The daylight saving status can be handled by postgres.
*/
const offsetNameQueryTpl = ctx => `
WITH __wd_tz AS
(
SELECT name
FROM pg_timezone_names
WHERE utc_offset = interval '${ctx.offset} hours'
ORDER BY CASE WHEN name LIKE 'Etc/GMT%' THEN 0 ELSE 1 END
LIMIT 1
),`;
/**
* Function to get the subquery that places each row in its bin depending on
* the aggregation. Since the data stored is in epoch we need to adapt it to
* our timezone so when calling date_trunc it falls into the correct bin
*/
function dataBucketsQuery(ctx) {
var condition_str = '';
if (ctx.start !== 0) {
condition_str = `WHERE ${ctx.column} >= to_timestamp(${ctx.start})`;
}
if (ctx.end !== 0) {
if (condition_str === '') {
condition_str = `WHERE ${ctx.column} <= to_timestamp(${ctx.end})`;
}
else {
condition_str += ` and ${ctx.column} <= to_timestamp(${ctx.end})`;
}
}
return `
__wd_buckets AS
(
SELECT
date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, ${ctx.column}::timestamptz)) as timestamp,
count(*) as freq,
${utils.countNULLs(ctx)} as nulls_count
FROM
(
${ctx.query}
) __source, __wd_tz
${condition_str}
GROUP BY timestamp, __wd_tz.name
),`;
}
/**
* Function that generates an array with all the possible bins between the
* start and end date. If not provided we use the min and max generated from
* the dataBucketsQuery
*/
function allBucketsArrayQuery(ctx) {
var extra_from = ``;
var series_start = ``;
var series_end = ``;
if (ctx.start === 0) {
extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
series_start = `min(__wd_buckets.timestamp)`;
} else {
series_start = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.start})))`;
}
if (ctx.end === 0) {
extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
series_end = `max(__wd_buckets.timestamp)`;
} else {
series_end = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.end})))`;
}
return `
__wd_all_buckets AS
(
SELECT ARRAY(
SELECT
generate_series(
${series_start},
${series_end},
interval '${ctx.interval}') as bin_start
FROM __wd_tz${extra_from}
) as bins
)`;
}
const dateIntervalQueryTpl = ctx => ` const dateIntervalQueryTpl = ctx => `
WITH WITH
@ -41,107 +138,6 @@ const dateIntervalQueryTpl = ctx => `
FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
`; `;
const nullsQueryTpl = ctx => `
__cdb_nulls AS (
SELECT
count(*) AS __cdb_nulls_count
FROM (${ctx.query}) __cdb_histogram_nulls
WHERE ${ctx.column} IS NULL
)
`;
const dateBasicsQueryTpl = ctx => `
__cdb_basics AS (
SELECT
max(date_part('epoch', ${ctx.column})) AS __cdb_max_val,
min(date_part('epoch', ${ctx.column})) AS __cdb_min_val,
avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
min(
date_trunc(
'${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
)
) AS __cdb_start_date,
max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date,
count(1) AS __cdb_total_rows
FROM (${ctx.query}) __cdb_basics_query
)
`;
const dateOverrideBasicsQueryTpl = ctx => `
__cdb_basics AS (
SELECT
max(${ctx.end})::float AS __cdb_max_val,
min(${ctx.start})::float AS __cdb_min_val,
avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
min(
date_trunc(
'${ctx.aggregation}',
TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}'
)
) AS __cdb_start_date,
max(
TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}'
) AS __cdb_end_date,
count(1) AS __cdb_total_rows
FROM (${ctx.query}) __cdb_basics_query
)
`;
const dateBinsQueryTpl = ctx => `
__cdb_bins AS (
SELECT
__cdb_bins_array,
ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number
FROM (
SELECT
ARRAY(
SELECT GENERATE_SERIES(
__cdb_start_date::timestamptz,
__cdb_end_date::timestamptz,
${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`}
)
) AS __cdb_bins_array
FROM __cdb_basics
) __cdb_bins_array_query
)
`;
const dateHistogramQueryTpl = ctx => `
SELECT
(__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
__cdb_bins_number AS bins_number,
__cdb_nulls_count AS nulls_count,
CASE WHEN __cdb_min_val = __cdb_max_val
THEN 0
ELSE GREATEST(
1,
LEAST(
WIDTH_BUCKET(
${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}',
__cdb_bins_array
),
__cdb_bins_number
)
) - 1
END AS bin,
min(
date_part(
'epoch',
date_trunc(
'${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
) AT TIME ZONE '${ctx.offset}'
)
)::numeric AS timestamp,
date_part('epoch', __cdb_start_date)::numeric AS timestamp_start,
min(date_part('epoch', ${ctx.column}))::numeric AS min,
max(date_part('epoch', ${ctx.column}))::numeric AS max,
avg(date_part('epoch', ${ctx.column}))::numeric AS avg,
count(*) AS freq
FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls
WHERE date_part('epoch', ${ctx.column}) IS NOT NULL
GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start
ORDER BY bin
`;
const MAX_INTERVAL_VALUE = 366; const MAX_INTERVAL_VALUE = 366;
@ -176,12 +172,21 @@ module.exports = class DateHistogram extends BaseHistogram {
_buildQueryTpl (ctx) { _buildQueryTpl (ctx) {
return ` return `
WITH ${offsetNameQueryTpl(ctx)}
${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)}, ${dataBucketsQuery(ctx)}
${dateBinsQueryTpl(ctx)}, ${allBucketsArrayQuery(ctx)}
${nullsQueryTpl(ctx)} SELECT
${dateHistogramQueryTpl(ctx)} array_position(__wd_all_buckets.bins, __wd_buckets.timestamp) - 1 as bin,
`; date_part('epoch', timezone(__wd_tz.name, __wd_buckets.timestamp)) AS timestamp,
__wd_buckets.freq as freq,
date_part('epoch', timezone(__wd_tz.name, (__wd_all_buckets.bins)[1])) as timestamp_start,
array_length(__wd_all_buckets.bins, 1) as bins_number,
date_part('epoch', interval '${ctx.interval}') as bin_width,
__wd_buckets.nulls_count as nulls_count
FROM __wd_buckets, __wd_all_buckets, __wd_tz
GROUP BY __wd_tz.name, __wd_all_buckets.bins, __wd_buckets.timestamp, __wd_buckets.nulls_count, __wd_buckets.freq
ORDER BY bin ASC;
`;
} }
_buildQuery (psql, override, callback) { _buildQuery (psql, override, callback) {
@ -204,6 +209,9 @@ module.exports = class DateHistogram extends BaseHistogram {
return null; return null;
} }
var interval = this._getAggregation(override) === 'quarter' ?
'3 months' : '1 ' + this._getAggregation(override);
const histogramSql = this._buildQueryTpl({ const histogramSql = this._buildQueryTpl({
override: override, override: override,
query: this.query, query: this.query,
@ -211,7 +219,8 @@ module.exports = class DateHistogram extends BaseHistogram {
aggregation: this._getAggregation(override), aggregation: this._getAggregation(override),
start: this._getBinStart(override), start: this._getBinStart(override),
end: this._getBinEnd(override), end: this._getBinEnd(override),
offset: this._parseOffset(override) offset: this._parseOffset(override),
interval: interval
}); });
debug(histogramSql); debug(histogramSql);
@ -264,8 +273,8 @@ module.exports = class DateHistogram extends BaseHistogram {
offset: this._getOffset(override), offset: this._getOffset(override),
timestamp_start: firstRow.timestamp_start, timestamp_start: firstRow.timestamp_start,
bin_width: firstRow.bin_width, bin_width: firstRow.bin_width || 0,
bins_count: firstRow.bins_number, bins_count: firstRow.bins_number || 0,
bins_start: firstRow.timestamp, bins_start: firstRow.timestamp,
nulls: firstRow.nulls_count, nulls: firstRow.nulls_count,
infinities: firstRow.infinities_count, infinities: firstRow.infinities_count,
@ -275,6 +284,10 @@ module.exports = class DateHistogram extends BaseHistogram {
} }
_getBuckets (result) { _getBuckets (result) {
result.rows.forEach(function(row) {
row.min = row.max = row.avg = row.timestamp;
});
return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp })); return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
} }

217
lib/cartodb/models/dataview/histograms/numeric-histogram.js
View File

@ -1,44 +1,25 @@
const BaseHistogram = require('./base-histogram'); const BaseHistogram = require('./base-histogram');
const debug = require('debug')('windshaft:dataview:numeric-histogram'); const debug = require('debug')('windshaft:dataview:numeric-histogram');
const utils = require('../../../utils/query-utils');
const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; /** Query to get min and max values from the query */
const irqQueryTpl = ctx => `
const filterOutSpecialNumericValues = ctx => `
${ctx.column} != 'infinity'::float
AND
${ctx.column} != '-infinity'::float
AND
${ctx.column} != 'NaN'::float
`;
const filteredQueryTpl = ctx => `
__cdb_filtered_source AS ( __cdb_filtered_source AS (
SELECT * SELECT *
FROM (${ctx.query}) __cdb_filtered_source_query FROM (${ctx.query}) __cdb_filtered_source_query
WHERE ${ctx.column} IS NOT NULL WHERE ${utils.handleFloatColumn(ctx)} IS NOT NULL
${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} ),
)
`;
const basicsQueryTpl = ctx => `
__cdb_basics AS ( __cdb_basics AS (
SELECT SELECT
max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val, max(${ctx.column}) AS __cdb_max_val,
avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows min(${ctx.column}) AS __cdb_min_val,
count(1) AS __cdb_total_rows
FROM __cdb_filtered_source FROM __cdb_filtered_source
) )
`; `;
const overrideBasicsQueryTpl = ctx => ` /* Query to calculate the number of bins (needs irqQueryTpl before it*/
__cdb_basics AS ( const binsQueryTpl = ctx => `
SELECT
max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val,
avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
FROM __cdb_filtered_source
)
`;
const iqrQueryTpl = ctx => `
__cdb_iqrange AS ( __cdb_iqrange AS (
SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
FROM ( FROM (
@ -49,10 +30,7 @@ const iqrQueryTpl = ctx => `
WHERE quartile = 1 or quartile = 3 WHERE quartile = 1 or quartile = 3
GROUP BY quartile GROUP BY quartile
) __cdb_iqr ) __cdb_iqr
) ),
`;
const binsQueryTpl = ctx => `
__cdb_bins AS ( __cdb_bins AS (
SELECT SELECT
CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
@ -70,83 +48,6 @@ const binsQueryTpl = ctx => `
) )
`; `;
const overrideBinsQueryTpl = ctx => `
__cdb_bins AS (
SELECT ${ctx.override.bins} AS __cdb_bins_number
)
`;
const nullsQueryTpl = ctx => `
__cdb_nulls AS (
SELECT
count(*) AS __cdb_nulls_count
FROM (${ctx.query}) __cdb_histogram_nulls
WHERE ${ctx.column} IS NULL
)
`;
const infinitiesQueryTpl = ctx => `
__cdb_infinities AS (
SELECT
count(*) AS __cdb_infinities_count
FROM (${ctx.query}) __cdb_infinities_query
WHERE
${ctx.column} = 'infinity'::float
OR
${ctx.column} = '-infinity'::float
)
`;
const nansQueryTpl = ctx => `
__cdb_nans AS (
SELECT
count(*) AS __cdb_nans_count
FROM (${ctx.query}) __cdb_nans_query
WHERE ${ctx.column} = 'NaN'::float
)
`;
const specialNumericValuesColumnDefinitionTpl = () => `
__cdb_infinities_count AS infinities_count,
__cdb_nans_count AS nans_count
`;
const specialNumericValuesCTETpl = () => `
__cdb_infinities, __cdb_nans
`;
const specialNumericValuesColumnTpl = () => `
infinities_count, nans_count
`;
const histogramQueryTpl = ctx => `
SELECT
(__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
__cdb_bins_number AS bins_number,
__cdb_nulls_count AS nulls_count,
${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''}
__cdb_avg_val AS avg_val,
CASE WHEN __cdb_min_val = __cdb_max_val
THEN 0
ELSE GREATEST(
1,
LEAST(
WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),
__cdb_bins_number
)
) - 1
END AS bin,
min(${ctx.column})::numeric AS min,
max(${ctx.column})::numeric AS max,
avg(${ctx.column})::numeric AS avg,
count(*) AS freq
FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins
${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''}
GROUP BY bin, bins_number, bin_width, nulls_count, avg_val
${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''}
ORDER BY bin
`;
const BIN_MIN_NUMBER = 6; const BIN_MIN_NUMBER = 6;
const BIN_MAX_NUMBER = 48; const BIN_MAX_NUMBER = 48;
@ -167,14 +68,14 @@ module.exports = class NumericHistogram extends BaseHistogram {
_buildQuery (psql, override, callback) { _buildQuery (psql, override, callback) {
const histogramSql = this._buildQueryTpl({ const histogramSql = this._buildQueryTpl({
override: override, column: this._columnType === 'date' ? utils.columnCastTpl({ column: this.column }) : this.column,
column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column,
isFloatColumn: this._columnType === 'float', isFloatColumn: this._columnType === 'float',
query: this.query, query: this.query,
start: this._getBinStart(override), start: this._getBinStart(override),
end: this._getBinEnd(override), end: this._getBinEnd(override),
bins: this._getBinsCount(override),
minBins: BIN_MIN_NUMBER, minBins: BIN_MIN_NUMBER,
maxBins: BIN_MAX_NUMBER, maxBins: BIN_MAX_NUMBER
}); });
debug(histogramSql); debug(histogramSql);
@ -182,19 +83,62 @@ module.exports = class NumericHistogram extends BaseHistogram {
return callback(null, histogramSql); return callback(null, histogramSql);
} }
/**
* ctx: Object with the following values
* ctx.column -- Column for the histogram
* ctx.isFloatColumn - Whether the column is float or not
* ctx.query -- Subquery to extract data
* ctx.start -- Start value for the bins. [>= end to force calculation]
* ctx.end -- End value for the bins.
* ctx.bins -- Numbers of bins to generate [<0 to force calculation]
* ctx.minBins - If !full min bins to calculate [Optional]
* ctx.maxBins - If !full max bins to calculate [Optional]
*/
_buildQueryTpl (ctx) { _buildQueryTpl (ctx) {
var extra_tables = ``;
var extra_queries = ``;
var extra_groupby = ``;
if (ctx.start >= ctx.end) {
ctx.end = `__cdb_basics.__cdb_max_val`;
ctx.start = `__cdb_basics.__cdb_min_val`;
extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`;
extra_tables = `, __cdb_basics`;
extra_queries = `WITH ${irqQueryTpl(ctx)}`;
}
if (ctx.bins <= 0) {
ctx.bins = `__cdb_bins.__cdb_bins_number`;
extra_groupby += `, __cdb_bins.__cdb_bins_number`;
extra_tables += `, __cdb_bins`;
extra_queries = `WITH ${irqQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`;
}
return ` return `
WITH ${extra_queries}
${filteredQueryTpl(ctx)}, SELECT
${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, (${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width,
${this._hasOverridenBins(ctx.override) ? ${ctx.bins} as bins_number,
overrideBinsQueryTpl(ctx) : ${utils.countNULLs(ctx)} AS nulls_count,
`${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` ${utils.countInfinites(ctx)} AS infinities_count,
}, ${utils.countNaNs(ctx)} AS nans_count,
${nullsQueryTpl(ctx)} min(${utils.handleFloatColumn(ctx)}) AS min,
${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} max(${utils.handleFloatColumn(ctx)}) AS max,
${histogramQueryTpl(ctx)} avg(${utils.handleFloatColumn(ctx)}) AS avg,
`; sum(CASE WHEN (${utils.handleFloatColumn(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq,
CASE WHEN ${ctx.start} = ${ctx.end}
THEN 0
ELSE GREATEST(1, LEAST(
${ctx.bins},
WIDTH_BUCKET(${utils.handleFloatColumn(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1
END AS bin
FROM
(
${ctx.query}
) __cdb_filtered_source_query${extra_tables}
GROUP BY bin${extra_groupby}
ORDER BY bin;`;
} }
_hasOverridenBins (override) { _hasOverridenBins (override) {
@ -204,14 +148,31 @@ module.exports = class NumericHistogram extends BaseHistogram {
_getSummary (result, override) { _getSummary (result, override) {
const firstRow = result.rows[0] || {}; const firstRow = result.rows[0] || {};
var total_nulls = 0;
var total_infinities = 0;
var total_nans = 0;
var total_avg = 0;
var total_count = 0;
result.rows.forEach(function(row) {
total_nulls += row.nulls_count;
total_infinities += row.infinities_count;
total_nans += row.nans_count;
total_avg += row.avg * row.freq;
total_count += row.freq;
});
if (total_count !== 0) {
total_avg /= total_count;
}
return { return {
bin_width: firstRow.bin_width, bin_width: firstRow.bin_width,
bins_count: firstRow.bins_number, bins_count: firstRow.bins_number,
bins_start: this._populateBinStart(firstRow, override), bins_start: this._populateBinStart(firstRow, override),
nulls: firstRow.nulls_count, nulls: total_nulls,
infinities: firstRow.infinities_count, infinities: total_infinities,
nans: firstRow.nans_count, nans: total_nans,
avg: firstRow.avg_val, avg: total_avg
}; };
} }

122
lib/cartodb/models/dataview/overviews/formula.js
View File

@ -1,55 +1,38 @@
var BaseOverviewsDataview = require('./base'); var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../formula'); var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overview'); var debug = require('debug')('windshaft:widget:formula:overview');
const utils = require('../../../utils/query-utils');
var dot = require('dot'); var dot = require('dot');
dot.templateSettings.strip = false; dot.templateSettings.strip = false;
var formulaQueryTpls = { const VALID_OPERATIONS = {
'count': dot.template([ count: true,
'SELECT', sum: true,
'sum(_feature_count) AS result,', avg: true
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
}; };
/** Formulae to calculate the end result using _feature_count from the overview table*/
function dataviewResult(ctx) {
switch (ctx.operation) {
case 'count':
return `sum(_feature_count)`;
case 'sum':
return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)`;
case 'avg':
return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)/sum(_feature_count) `;
}
return `${ctx.operation}(${utils.handleFloatColumn(ctx)})`;
}
const formulaQueryTpl = ctx =>
`SELECT
${dataviewResult(ctx)} AS result,
${utils.countNULLs(ctx)} AS nulls_count
${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
FROM (${ctx.query}) __cdb_formula`;
function Formula(query, options, queryRewriter, queryRewriteData, params, queries) { function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.column = options.column || '1'; this.column = options.column || '1';
@ -65,36 +48,31 @@ module.exports = Formula;
Formula.prototype.sql = function (psql, override, callback) { Formula.prototype.sql = function (psql, override, callback) {
var self = this; var self = this;
var formulaQueryTpl = formulaQueryTpls[this.operation]; if (!VALID_OPERATIONS[this.operation]) {
return this.defaultSql(psql, override, callback);
if (formulaQueryTpl) {
// supported formula for use with overviews
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var formulaSql = formulaQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: this.rewrittenQuery(this.query),
_operation: this.operation,
_column: this.column
});
callback = callback || override;
debug(formulaSql);
return callback(null, formulaSql);
} }
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
// default behaviour var formulaSql = formulaQueryTpl({
return this.defaultSql(psql, override, callback); isFloatColumn: this._isFloatColumn,
query: this.rewrittenQuery(this.query),
operation: this.operation,
column: this.column
});
callback = callback || override;
debug(formulaSql);
return callback(null, formulaSql);
}; };

33
lib/cartodb/utils/query-utils.js
View File

@ -22,5 +22,36 @@ module.exports.extractTableNames = function extractTableNames(query) {
}; };
module.exports.getQueryRowCount = function getQueryRowEstimation(query) { module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount(\'' + query + '\') as rows'; return 'select CDB_EstimateRowCount($$' + query + '$$) as rows';
}; };
/** Cast the column to epoch */
module.exports.columnCastTpl = function columnCastTpl(ctx) {
return `date_part('epoch', ${ctx.column})`;
};
/** If the column type is float, ignore any non numeric result (infinity / NaN) */
module.exports.handleFloatColumn = function handleFloatColumn(ctx) {
return `${!ctx.isFloatColumn ? `${ctx.column}` :
`nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)`
}`;
};
/** Count NULL appearances */
module.exports.countNULLs= function countNULLs(ctx) {
return `sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END)`;
};
/** Count only infinity (positive and negative) appearances */
module.exports.countInfinites = function countInfinites(ctx) {
return `${!ctx.isFloatColumn ? `0` :
`sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)`
}`;
};
/** Count only NaNs appearances*/
module.exports.countNaNs = function countNaNs(ctx) {
return `${!ctx.isFloatColumn ? `0` :
`sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)`
}`;
};

6
package.json
View File

@ -1,7 +1,7 @@
{ {
"private": true, "private": true,
"name": "windshaft-cartodb", "name": "windshaft-cartodb",
"version": "4.0.2", "version": "4.2.1",
"description": "A map tile server for CartoDB", "description": "A map tile server for CartoDB",
"keywords": [ "keywords": [
"cartodb" "cartodb"
@ -42,9 +42,9 @@
"semver": "~5.3.0", "semver": "~5.3.0",
"step": "~0.0.6", "step": "~0.0.6",
"step-profiler": "~0.3.0", "step-profiler": "~0.3.0",
"turbo-carto": "0.20.1", "turbo-carto": "0.20.2",
"underscore": "~1.6.0", "underscore": "~1.6.0",
"windshaft": "4.0.1", "windshaft": "4.1.0",
"yargs": "~5.0.0" "yargs": "~5.0.0"
}, },
"devDependencies": { "devDependencies": {

80
test/acceptance/dataviews/histogram.js
View File

@ -186,7 +186,7 @@ describe('histogram-dataview for date column type', function() {
}, },
minute_histogram: { minute_histogram: {
source: { source: {
id: 'minute-histogram-source' id: 'minute-histogram-source-tz'
}, },
type: 'histogram', type: 'histogram',
options: { options: {
@ -214,8 +214,8 @@ describe('histogram-dataview for date column type', function() {
"params": { "params": {
"query": [ "query": [
"select null::geometry the_geom_webmercator, date AS d", "select null::geometry the_geom_webmercator, date AS d",
"from generate_series(", "from generate_series('2007-02-15 01:00:00+00'::timestamptz,",
"'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", "'2008-04-09 01:00:00+00'::timestamptz, '1 day'::interval",
") date" ") date"
].join(' ') ].join(' ')
} }
@ -233,13 +233,13 @@ describe('histogram-dataview for date column type', function() {
} }
}, },
{ {
"id": "minute-histogram-source", "id": "minute-histogram-source-tz",
"type": "source", "type": "source",
"params": { "params": {
"query": [ "query": [
"select null::geometry the_geom_webmercator, date AS d", "select null::geometry the_geom_webmercator, date AS d",
"from generate_series(", "from generate_series('2007-02-15 23:50:00+00'::timestamptz,",
"'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval", "'2007-02-16 00:10:00+00'::timestamptz, '1 minute'::interval",
") date" ") date"
].join(' ') ].join(' ')
} }
@ -256,6 +256,7 @@ describe('histogram-dataview for date column type', function() {
}]; }];
dateHistogramsUseCases.forEach(function (test) { dateHistogramsUseCases.forEach(function (test) {
it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) {
var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes
@ -323,7 +324,7 @@ describe('histogram-dataview for date column type', function() {
assert.ok(!err, err); assert.ok(!err, err);
assert.equal(dataview.type, 'histogram'); assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 6); assert.equal(dataview.bins_count, 6);
dataview.bins.forEach(function (bin) { dataview.bins.forEach(function (bin) {
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
}); });
@ -335,7 +336,7 @@ describe('histogram-dataview for date column type', function() {
it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) { it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) {
var params = { var params = {
start: -2145916800, start: -2145916800,
end: 1009843199 end: 1193792400
}; };
this.testClient = new TestClient(mapConfig, 1234); this.testClient = new TestClient(mapConfig, 1234);
@ -348,27 +349,6 @@ describe('histogram-dataview for date column type', function() {
}); });
}); });
it('should return same histogram ' + test.desc, function (done) {
var params = {
start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum)
end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum)
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) {
assert.ok(!err, err);
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) {
assert.ok(!err, err);
assert.deepEqual(dataview, filteredDataview);
done();
});
});
});
it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) { it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) {
var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time)
var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time)
@ -533,6 +513,26 @@ describe('histogram-dataview for date column type', function() {
}); });
}); });
it('should return same histogram ', function (done) {
var params = {
start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum)
end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum)
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_tz', {}, function (err, dataview) {
assert.ok(!err, err);
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_tz', params, function (err, filteredDataview) {
assert.ok(!err, err);
assert.deepEqual(dataview, filteredDataview);
done();
});
});
});
it('should find the best aggregation (automatic mode) to build the histogram', function (done) { it('should find the best aggregation (automatic mode) to build the histogram', function (done) {
var params = {}; var params = {};
this.testClient = new TestClient(mapConfig, 1234); this.testClient = new TestClient(mapConfig, 1234);
@ -640,7 +640,7 @@ describe('histogram-dataview for date column type', function() {
var dataviewWithDailyAggFixture = { var dataviewWithDailyAggFixture = {
aggregation: 'day', aggregation: 'day',
bin_width: 600, bin_width: 86400,
bins_count: 2, bins_count: 2,
bins_start: 1171497600, bins_start: 1171497600,
timestamp_start: 1171497600, timestamp_start: 1171497600,
@ -650,17 +650,17 @@ describe('histogram-dataview for date column type', function() {
[{ [{
bin: 0, bin: 0,
timestamp: 1171497600, timestamp: 1171497600,
min: 1171583400, min: 1171497600,
max: 1171583940, max: 1171497600,
avg: 1171583670, avg: 1171497600,
freq: 10 freq: 10
}, },
{ {
bin: 1, bin: 1,
timestamp: 1171584000, timestamp: 1171584000,
min: 1171584000, min: 1171584000,
max: 1171584600, max: 1171584000,
avg: 1171584300, avg: 1171584000,
freq: 11 freq: 11
}], }],
type: 'histogram' type: 'histogram'
@ -687,19 +687,19 @@ describe('histogram-dataview for date column type', function() {
var dataviewWithDailyAggAndOffsetFixture = { var dataviewWithDailyAggAndOffsetFixture = {
aggregation: 'day', aggregation: 'day',
bin_width: 1200, bin_width: 86400,
bins_count: 1, bins_count: 1,
bins_start: 1171501200, bins_start: 1171501200,
timestamp_start: 1171497600, timestamp_start: 1171501200,
nulls: 0, nulls: 0,
offset: -3600, offset: -3600,
bins: bins:
[{ [{
bin: 0, bin: 0,
timestamp: 1171501200, timestamp: 1171501200,
min: 1171583400, min: 1171501200,
max: 1171584600, max: 1171501200,
avg: 1171584000, avg: 1171501200,
freq: 21 freq: 21
}], }],
type: 'histogram' type: 'histogram'

6
test/acceptance/multilayer.js
View File

@ -1163,8 +1163,12 @@ describe(suiteName, function() {
); );
}); });
// WARN: MapConfig with mapnik layer and no cartocss it's valid since
// vector & raster aggregation project, now we can request MVT format w/o defining styles
// for the layer.
// See https://github.com/CartoDB/Windshaft-cartodb/issues/133 // See https://github.com/CartoDB/Windshaft-cartodb/issues/133
it("MapConfig with mapnik layer and no cartocss", function(done) { it.skip("MapConfig with mapnik layer and no cartocss", function(done) {
var layergroup = { var layergroup = {
version: '1.0.0', version: '1.0.0',

2
test/acceptance/mvt.js
View File

@ -504,4 +504,4 @@ return function () {
} }
}; };
} }

221
test/acceptance/vector-layergroup.js Normal file
View File

@ -0,0 +1,221 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const serverOptions = require('../../lib/cartodb/server_options');
const POINTS_SQL_1 = `
select
st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom,
st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator,
x as value
from generate_series(-3, 3) x
`;
const POINTS_SQL_2 = `
select
st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326) as the_geom,
st_transform(st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326), 3857) as the_geom_webmercator,
x as value
from generate_series(-3, 3) x
`;
function createVectorLayergroup () {
return {
version: '1.6.0',
layers: [
{
type: 'cartodb',
options: {
sql: POINTS_SQL_1
}
},
{
type: 'cartodb',
options: {
sql: POINTS_SQL_2
}
}
]
};
}
const INCOMPATIBLE_LAYERS_ERROR = {
"errors": [
"The `mapnik` or `cartodb` layers must be consistent:" +
" `cartocss` option is either present or voided in all layers. Mixing is not allowed."
],
"errors_with_context":[
{
"type":"mapconfig",
"message": "The `mapnik` or `cartodb` layers must be consistent:" +
" `cartocss` option is either present or voided in all layers. Mixing is not allowed."
}
]
};
const INVALID_FORMAT_ERROR = {
"errors": [
"Unsupported format: 'cartocss' option is missing for png"
],
"errors_with_context":[
{
"type": "tile",
"message": "Unsupported format: 'cartocss' option is missing for png"
}
]
};
const suites = [{
desc: 'mvt (mapnik)',
usePostGIS: false
}];
if (process.env.POSTGIS_VERSION === '2.4') {
suites.push({
desc: 'mvt (postgis)',
usePostGIS: true
});
}
suites.forEach((suite) => {
const { desc, usePostGIS } = suite;
describe(desc, function () {
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
before(function () {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
});
after(function (){
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
});
describe('vector-layergroup', function () {
beforeEach(function () {
this.mapConfig = createVectorLayergroup();
this.testClient = new TestClient(this.mapConfig);
});
afterEach(function (done) {
this.testClient.drain(done);
});
it('should get vector tiles from layergroup with layers w/o cartocss', function (done) {
this.testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, tile) => {
if (err) {
return done(err);
}
assert.equal(tile.tileSize, 4096);
assert.equal(tile.z, 0);
assert.equal(tile.x, 0);
assert.equal(tile.y, 0);
const layer0 = JSON.parse(tile.toGeoJSONSync(0));
assert.equal(layer0.name, 'layer0');
assert.equal(layer0.features[0].type, 'Feature');
assert.equal(layer0.features[0].geometry.type, 'Point');
const layer1 = JSON.parse(tile.toGeoJSONSync(1));
assert.equal(layer1.name, 'layer1');
assert.equal(layer1.features[0].type, 'Feature');
assert.equal(layer1.features[0].geometry.type, 'Point');
done();
});
});
it('should get vector tiles from specific layer (layer0)', function (done) {
this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, (err, res, tile) => {
if (err) {
return done(err);
}
assert.equal(tile.tileSize, 4096);
assert.equal(tile.z, 0);
assert.equal(tile.x, 0);
assert.equal(tile.y, 0);
const layer = JSON.parse(tile.toGeoJSONSync(0));
assert.equal(layer.name, 'layer0');
assert.equal(layer.features[0].type, 'Feature');
assert.equal(layer.features[0].geometry.type, 'Point');
done();
});
});
it('should get vector tiles from specific layer (layer1)', function (done) {
this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 1 }, (err, res, tile) => {
if (err) {
return done(err);
}
assert.equal(tile.tileSize, 4096);
assert.equal(tile.z, 0);
assert.equal(tile.x, 0);
assert.equal(tile.y, 0);
const layer = JSON.parse(tile.toGeoJSONSync(0));
assert.equal(layer.name, 'layer1');
assert.equal(layer.features[0].type, 'Feature');
assert.equal(layer.features[0].geometry.type, 'Point');
done();
});
});
it('should fail when the format requested is not mvt', function (done) {
const options = {
format: 'png',
response: {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, options, (err, res, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, INVALID_FORMAT_ERROR);
done();
});
});
it('should fail when the map-config mix layers with and without cartocss', function (done) {
const response = {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
const cartocss = `#layer0 { marker-fill: red; marker-width: 10; }`;
const cartocssVersion = '2.3.0';
this.testClient.mapConfig.layers[0].options.cartocss = cartocss;
this.testClient.mapConfig.layers[0].options.cartocss_version = cartocssVersion;
this.testClient.getLayergroup(response, (err, body) => {
if (err) {
return done(err);
}
assert.deepEqual(body, INCOMPATIBLE_LAYERS_ERROR);
done();
});
});
});
});
});

101
yarn.lock
View File

@ -240,9 +240,9 @@ carto@0.16.3:
semver "^5.1.0" semver "^5.1.0"
yargs "^4.2.0" yargs "^4.2.0"
carto@CartoDB/carto#0.15.1-cdb1: "carto@github:cartodb/carto#0.15.1-cdb1":
version "0.15.1-cdb1" version "0.15.1-cdb1"
resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
dependencies: dependencies:
mapnik-reference "~6.0.2" mapnik-reference "~6.0.2"
optimist "~0.6.0" optimist "~0.6.0"
@ -484,6 +484,10 @@ destroy@~1.0.4:
version "1.0.4" version "1.0.4"
resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80"
detect-libc@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
diff@3.2.0: diff@3.2.0:
version "3.2.0" version "3.2.0"
resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9" resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9"
@ -746,7 +750,7 @@ gdal@~0.9.2:
nan "~2.6.2" nan "~2.6.2"
node-pre-gyp "~0.6.36" node-pre-gyp "~0.6.36"
generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: generic-pool@2.4.3:
version "2.4.3" version "2.4.3"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff"
@ -758,6 +762,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1:
version "2.2.2" version "2.2.2"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be"
generic-pool@~2.4.0, generic-pool@~2.4.1:
version "2.4.6"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7"
get-caller-file@^1.0.1: get-caller-file@^1.0.1:
version "1.0.2" version "1.0.2"
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5"
@ -768,7 +776,7 @@ getpass@^0.1.1:
dependencies: dependencies:
assert-plus "^1.0.0" assert-plus "^1.0.0"
glob@7.1.1, glob@^7.0.5, glob@^7.1.1: glob@7.1.1, glob@^7.1.1:
version "7.1.1" version "7.1.1"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8"
dependencies: dependencies:
@ -799,6 +807,17 @@ glob@^6.0.1:
once "^1.3.0" once "^1.3.0"
path-is-absolute "^1.0.0" path-is-absolute "^1.0.0"
glob@^7.0.5:
version "7.1.2"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.0.4"
once "^1.3.0"
path-is-absolute "^1.0.0"
graceful-fs@^4.1.2: graceful-fs@^4.1.2:
version "4.1.11" version "4.1.11"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
@ -807,9 +826,9 @@ graceful-fs@^4.1.2:
version "1.0.1" version "1.0.1"
resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725"
grainstore@~1.6.0: grainstore@1.7.0:
version "1.6.4" version "1.7.0"
resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.4.tgz#617b93c5e2de8f544375202da89b9208a8b3d762" resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.7.0.tgz#28d78895c82e6201f7d0ff63af1056f3c0fda0d3"
dependencies: dependencies:
carto "0.16.3" carto "0.16.3"
debug "~3.1.0" debug "~3.1.0"
@ -963,8 +982,8 @@ inherits@2, inherits@2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
ini@~1.3.0: ini@~1.3.0:
version "1.3.4" version "1.3.5"
resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.4.tgz#0537cb79daf59b59a1a517dff706c86ec039162e" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927"
invert-kv@^1.0.0: invert-kv@^1.0.0:
version "1.0.0" version "1.0.0"
@ -1300,7 +1319,7 @@ mime@~1.3.4:
version "1.3.6" version "1.3.6"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0"
"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2:
version "3.0.4" version "3.0.4"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
dependencies: dependencies:
@ -1360,10 +1379,14 @@ mv@~2:
ncp "~2.0.0" ncp "~2.0.0"
rimraf "~2.4.0" rimraf "~2.4.0"
nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.7.0: nan@^2.0.8, nan@^2.3.4, nan@~2.7.0:
version "2.7.0" version "2.7.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46"
nan@^2.4.0:
version "2.8.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a"
nan@~2.4.0: nan@~2.4.0:
version "2.4.0" version "2.4.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232"
@ -1392,9 +1415,10 @@ nock@~2.11.0:
propagate "0.3.x" propagate "0.3.x"
node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38:
version "0.6.38" version "0.6.39"
resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.38.tgz#e92a20f83416415bb4086f6d1fb78b3da73d113d" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649"
dependencies: dependencies:
detect-libc "^1.0.2"
hawk "3.1.3" hawk "3.1.3"
mkdirp "^0.5.1" mkdirp "^0.5.1"
nopt "^4.0.1" nopt "^4.0.1"
@ -1449,10 +1473,14 @@ oauth-sign@~0.8.1, oauth-sign@~0.8.2:
version "0.8.2" version "0.8.2"
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
object-assign@4.1.0, object-assign@^4.1.0: object-assign@4.1.0:
version "4.1.0" version "4.1.0"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0"
object-assign@^4.1.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
object-keys@~0.4.0: object-keys@~0.4.0:
version "0.4.0" version "0.4.0"
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336"
@ -1721,8 +1749,8 @@ raw-body@2.3.2:
unpipe "1.0.0" unpipe "1.0.0"
rc@^1.1.7: rc@^1.1.7:
version "1.2.1" version "1.2.2"
resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.1.tgz#2e03e8e42ee450b8cb3dce65be1bf8974e1dfd95" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.2.tgz#d8ce9cb57e8d64d9c7badd9876c7c34cbe3c7077"
dependencies: dependencies:
deep-extend "~0.4.0" deep-extend "~0.4.0"
ini "~1.3.0" ini "~1.3.0"
@ -1744,7 +1772,7 @@ read-pkg@^1.0.0:
normalize-package-data "^2.3.2" normalize-package-data "^2.3.2"
path-type "^1.0.0" path-type "^1.0.0"
readable-stream@1.1, readable-stream@~1.1.9: readable-stream@1.1:
version "1.1.13" version "1.1.13"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e"
dependencies: dependencies:
@ -1774,6 +1802,15 @@ readable-stream@~1.0.2:
isarray "0.0.1" isarray "0.0.1"
string_decoder "~0.10.x" string_decoder "~0.10.x"
readable-stream@~1.1.9:
version "1.1.14"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.1"
isarray "0.0.1"
string_decoder "~0.10.x"
redis-mpool@0.4.1, redis-mpool@~0.4.1: redis-mpool@0.4.1, redis-mpool@~0.4.1:
version "0.4.1" version "0.4.1"
resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80"
@ -1883,7 +1920,7 @@ safe-json-stringify@~1:
version "1.0.4" version "1.0.4"
resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911"
"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: "semver@2 || 3 || 4 || 5", semver@~5.3.0:
version "5.3.0" version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
@ -1891,6 +1928,10 @@ semver@4.3.2:
version "4.3.2" version "4.3.2"
resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7"
semver@^5.1.0, semver@^5.3.0:
version "5.4.1"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e"
semver@~4.3.3: semver@~4.3.3:
version "4.3.6" version "4.3.6"
resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da"
@ -1996,10 +2037,14 @@ speedometer@~0.1.2:
version "0.1.4" version "0.1.4"
resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d"
sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: sphericalmercator@1.0.4:
version "1.0.4" version "1.0.4"
resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e"
sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980"
split@^1.0.0: split@^1.0.0:
version "1.0.1" version "1.0.1"
resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9"
@ -2114,8 +2159,8 @@ supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3:
has-flag "^1.0.0" has-flag "^1.0.0"
tar-pack@^3.4.0: tar-pack@^3.4.0:
version "3.4.0" version "3.4.1"
resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.0.tgz#23be2d7f671a8339376cbdb0b8fe3fdebf317984" resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.1.tgz#e1dbc03a9b9d3ba07e896ad027317eb679a10a1f"
dependencies: dependencies:
debug "^2.2.0" debug "^2.2.0"
fstream "^1.0.10" fstream "^1.0.10"
@ -2190,9 +2235,9 @@ tunnel-agent@^0.6.0:
dependencies: dependencies:
safe-buffer "^5.0.1" safe-buffer "^5.0.1"
turbo-carto@0.20.1: turbo-carto@0.20.2:
version "0.20.1" version "0.20.2"
resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.1.tgz#e9f5fa1408d9d4325a1e79333e6d242170f89e6d" resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.2.tgz#2b737597a65c2918432f70ea414f12fbec2b6a6f"
dependencies: dependencies:
cartocolor "4.0.0" cartocolor "4.0.0"
colorbrewer "1.0.0" colorbrewer "1.0.0"
@ -2314,9 +2359,9 @@ window-size@^0.2.0:
version "0.2.0" version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@4.0.1: windshaft@^4.1.0:
version "4.0.1" version "4.1.0"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.1.tgz#16230a0b5b28f08a443491339c37637e3caefa52" resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.1.0.tgz#dc17c8369570c305171d1ab5ca130369bba04d58"
dependencies: dependencies:
abaculus cartodb/abaculus#2.0.3-cdb1 abaculus cartodb/abaculus#2.0.3-cdb1
canvas cartodb/node-canvas#1.6.2-cdb2 canvas cartodb/node-canvas#1.6.2-cdb2
@ -2324,7 +2369,7 @@ windshaft@4.0.1:
cartodb-psql "^0.10.1" cartodb-psql "^0.10.1"
debug "^3.1.0" debug "^3.1.0"
dot "~1.0.2" dot "~1.0.2"
grainstore "~1.6.0" grainstore "1.7.0"
mapnik "3.5.14" mapnik "3.5.14"
queue-async "~1.0.7" queue-async "~1.0.7"
redis-mpool "0.4.1" redis-mpool "0.4.1"