From 5ce419d86392ec9f4cdb736d18cea6c8ac90437c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 4 Sep 2017 18:42:30 +0200 Subject: [PATCH 001/177] Use template string for dateIntervalQueryTpl --- lib/cartodb/models/dataview/histogram.js | 78 ++++++++++++------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c27c696f..1a526637 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -7,45 +7,45 @@ dot.templateSettings.strip = false; var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); -var dateIntervalQueryTpl = dot.template([ - 'WITH', - '__cdb_dates AS (', - ' SELECT', - ' MAX({{=it.column}}::timestamp) AS __cdb_end,', - ' MIN({{=it.column}}::timestamp) AS __cdb_start', - ' FROM ({{=it.query}}) __cdb_source', - '),', - '__cdb_interval_in_days AS (', - ' SELECT' , - ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', - ' FROM __cdb_dates', - '),', - '__cdb_interval_in_hours AS (', - ' SELECT', - ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', - ' FROM __cdb_interval_in_days, __cdb_dates', - '),', - '__cdb_interval_in_minutes AS (', - ' SELECT', - ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', - ' FROM __cdb_interval_in_hours, __cdb_dates', - '),', - '__cdb_interval_in_seconds AS (', - ' SELECT', - ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', - ' FROM __cdb_interval_in_minutes, __cdb_dates', - ')', - 'SELECT', - ' ROUND(__cdb_days / 365) AS year,', - ' ROUND(__cdb_days / 90) AS quarter,', - ' ROUND(__cdb_days / 30) AS month,', - ' ROUND(__cdb_days / 7) AS week,', - ' __cdb_days AS day,', - ' __cdb_hours AS hour,', - ' __cdb_minutes AS minute,', - ' __cdb_seconds AS second', - 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' -].join('\n')); +const dateIntervalQueryTpl = ctx => ` + WITH + __cdb_dates AS ( + SELECT + MAX(${ctx.column}::timestamp) AS __cdb_end + MIN(${ctx.column}::timestamp) AS __cdb_start + FROM (${ctx.query}) __cdb_source + ), + __cdb_interval_in_days AS ( + SELECT + DATE_PART(\day\ __cdb_end - __cdb_start) AS __cdb_days + FROM __cdb_dates + ), + __cdb_interval_in_hours AS ( + SELECT + __cdb_days * 24 + DATE_PART(\hour\ __cdb_end - __cdb_start) AS __cdb_hours + FROM __cdb_interval_in_days __cdb_dates + ), + __cdb_interval_in_minutes AS ( + SELECT + __cdb_hours * 60 + DATE_PART(\minute\ __cdb_end - __cdb_start) AS __cdb_minutes + FROM __cdb_interval_in_hours __cdb_dates + ), + __cdb_interval_in_seconds AS ( + SELECT + __cdb_minutes * 60 + DATE_PART(\second\ __cdb_end - __cdb_start) AS __cdb_seconds + FROM __cdb_interval_in_minutes __cdb_dates + ), + SELECT + ROUND(__cdb_days / 365) AS year + ROUND(__cdb_days / 90) AS quarter + ROUND(__cdb_days / 30) AS month + ROUND(__cdb_days / 7) AS week + __cdb_days AS day + __cdb_hours AS hour + __cdb_minutes AS minute + __cdb_seconds AS second + FROM __cdb_interval_in_days __cdb_interval_in_hours __cdb_interval_in_minutes __cdb_interval_in_seconds +`; var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; From 6373fe86521405a30ea870a257ac22dbeacbc91b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 4 Sep 2017 19:01:58 +0200 Subject: [PATCH 002/177] Use template string for filteredQueryTpl --- lib/cartodb/models/dataview/histogram.js | 31 ++++++++++++------------ 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 1a526637..60e10418 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -51,21 +51,22 @@ var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; -var filteredQueryTpl = dot.template([ - '__cdb_filtered_source AS (', - ' SELECT *', - ' FROM ({{=it._query}}) __cdb_filtered_source_query', - ' WHERE', - ' {{=it._column}} IS NOT NULL', - ' {{?it._isFloatColumn}}AND', - ' {{=it._column}} != \'infinity\'::float', - ' AND', - ' {{=it._column}} != \'-infinity\'::float', - ' AND', - ' {{=it._column}} != \'NaN\'::float{{?}}', - ')' -].join(' \n')); - +const filteredQueryTpl = ctx => ` + __cdb_filtered_source AS ( + SELECT * + FROM (${ctx._query}) __cdb_filtered_source_query + WHERE + ${ctx._column} IS NOT NULL + ${ctx._isFloatColumn ? ` + AND + ${ctx._column} != 'infinity'::float + AND + ${ctx._column} != '-infinity'::float + AND + ${ctx._column} != 'NaN'::float + ` : ''} + ) +` var basicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', From c5806005908b3bbe471d9d863ca145f434adbb6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 10:36:18 +0200 Subject: [PATCH 003/177] Extract template to filter out special numeric values --- lib/cartodb/models/dataview/histogram.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 60e10418..3f31bea8 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -51,22 +51,23 @@ var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +const filterOutSpecialNumericValues = ctx => ` + ${ctx._column} != 'infinity'::float + AND + ${ctx._column} != '-infinity'::float + AND + ${ctx._column} != 'NaN'::float +`; + const filteredQueryTpl = ctx => ` __cdb_filtered_source AS ( SELECT * FROM (${ctx._query}) __cdb_filtered_source_query - WHERE - ${ctx._column} IS NOT NULL - ${ctx._isFloatColumn ? ` - AND - ${ctx._column} != 'infinity'::float - AND - ${ctx._column} != '-infinity'::float - AND - ${ctx._column} != 'NaN'::float - ` : ''} + WHERE ${ctx._column} IS NOT NULL + ${ctx._isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} ) -` +`; + var basicsQueryTpl = dot.template([ '__cdb_basics AS (', ' SELECT', From 00741bc0a4a9b773d781063b168f9f2e8e9e86a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 10:41:48 +0200 Subject: [PATCH 004/177] Use template string for basicsQueryTpl --- lib/cartodb/models/dataview/histogram.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 3f31bea8..11fe7bda 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -68,14 +68,14 @@ const filteredQueryTpl = ctx => ` ) `; -var basicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', - ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', - ' FROM __cdb_filtered_source', - ')' -].join(' \n')); +const basicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx._column}) AS __cdb_max_val, min(${ctx._column}) AS __cdb_min_val, + avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; var overrideBasicsQueryTpl = dot.template([ '__cdb_basics AS (', From 37b137676703e70aafe829415228cde18cd3bffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 11:26:27 +0200 Subject: [PATCH 005/177] Fix bad find & replace --- lib/cartodb/models/dataview/histogram.js | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 11fe7bda..ca1b9151 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -11,40 +11,40 @@ const dateIntervalQueryTpl = ctx => ` WITH __cdb_dates AS ( SELECT - MAX(${ctx.column}::timestamp) AS __cdb_end + MAX(${ctx.column}::timestamp) AS __cdb_end, MIN(${ctx.column}::timestamp) AS __cdb_start FROM (${ctx.query}) __cdb_source ), __cdb_interval_in_days AS ( SELECT - DATE_PART(\day\ __cdb_end - __cdb_start) AS __cdb_days + DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days FROM __cdb_dates ), __cdb_interval_in_hours AS ( SELECT - __cdb_days * 24 + DATE_PART(\hour\ __cdb_end - __cdb_start) AS __cdb_hours - FROM __cdb_interval_in_days __cdb_dates + __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours + FROM __cdb_interval_in_days, __cdb_dates ), __cdb_interval_in_minutes AS ( SELECT - __cdb_hours * 60 + DATE_PART(\minute\ __cdb_end - __cdb_start) AS __cdb_minutes - FROM __cdb_interval_in_hours __cdb_dates + __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes + FROM __cdb_interval_in_hours, __cdb_dates ), __cdb_interval_in_seconds AS ( SELECT - __cdb_minutes * 60 + DATE_PART(\second\ __cdb_end - __cdb_start) AS __cdb_seconds - FROM __cdb_interval_in_minutes __cdb_dates - ), + __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds + FROM __cdb_interval_in_minutes, __cdb_dates + ) SELECT - ROUND(__cdb_days / 365) AS year - ROUND(__cdb_days / 90) AS quarter - ROUND(__cdb_days / 30) AS month - ROUND(__cdb_days / 7) AS week - __cdb_days AS day - __cdb_hours AS hour - __cdb_minutes AS minute + ROUND(__cdb_days / 365) AS year, + ROUND(__cdb_days / 90) AS quarter, + ROUND(__cdb_days / 30) AS month, + ROUND(__cdb_days / 7) AS week, + __cdb_days AS day, + __cdb_hours AS hour, + __cdb_minutes AS minute, __cdb_seconds AS second - FROM __cdb_interval_in_days __cdb_interval_in_hours __cdb_interval_in_minutes __cdb_interval_in_seconds + FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds `; var MAX_INTERVAL_VALUE = 366; From 5d413ac1f98d806f754d3f0a7d4d264551496b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 11:26:38 +0200 Subject: [PATCH 006/177] Use template string for overrideBasicsQueryTpl --- lib/cartodb/models/dataview/histogram.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index ca1b9151..b00b1e1a 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -77,14 +77,14 @@ const basicsQueryTpl = ctx => ` ) `; -var overrideBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', - ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', - ' FROM __cdb_filtered_source', - ')' -].join('\n')); +const overrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx._end}) AS __cdb_max_val, min(${ctx._start}) AS __cdb_min_val, + avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; var iqrQueryTpl = dot.template([ '__cdb_iqrange AS (', From 151bdec1fde503d8b2d8e781cec47c69ec0d93bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 12:16:38 +0200 Subject: [PATCH 007/177] Use template string for overrideBinsQueryTpl --- lib/cartodb/models/dataview/histogram.js | 69 ++++++++++++------------ 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b00b1e1a..5eec9713 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -86,42 +86,43 @@ const overrideBasicsQueryTpl = ctx => ` ) `; -var iqrQueryTpl = dot.template([ - '__cdb_iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', - ' FROM (', - ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', - ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', - ' ) AS quartile', - ' FROM __cdb_filtered_source) _cdb_quartiles', - ' WHERE quartile = 1 or quartile = 3', - ' GROUP BY quartile', - ' ) __cdb_iqr', - ')' -].join('\n')); +const iqrQueryTpl = ctx => ` + __cdb_iqrange AS ( + SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr + FROM ( + SELECT quartile, max(_cdb_iqr_column) AS quartile_max from ( + SELECT ${ctx._column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx._column} + ) AS quartile + FROM __cdb_filtered_source) _cdb_quartiles + WHERE quartile = 1 or quartile = 3 + GROUP BY quartile + ) __cdb_iqr + ) +`; -var binsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', - ' THEN 1', - ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', - ' LEAST(', - ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', - ' {{=it._maxBins}}', - ' )', - ' )', - ' END AS __cdb_bins_number', - ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', - ' LIMIT 1', - ')' -].join('\n')); +const binsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 + THEN 1 + ELSE GREATEST( + LEAST(${ctx._minBins}, CAST(__cdb_total_rows AS INT)), + LEAST( + CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT), + ${ctx._maxBins} + ) + ) + END AS __cdb_bins_number + FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source + LIMIT 1 + ) +`; -var overrideBinsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT {{=it._bins}} AS __cdb_bins_number', - ')' -].join('\n')); +const overrideBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT ${ctx._bins} AS __cdb_bins_number + ) +`; var nullsQueryTpl = dot.template([ '__cdb_nulls AS (', From f1018f3272cbd226723adb5b2671a95c5e393151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 12:18:42 +0200 Subject: [PATCH 008/177] Use template string for nullsQueryTpl --- lib/cartodb/models/dataview/histogram.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5eec9713..35c9b43c 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -124,14 +124,14 @@ const overrideBinsQueryTpl = ctx => ` ) `; -var nullsQueryTpl = dot.template([ - '__cdb_nulls AS (', - ' SELECT', - ' count(*) AS __cdb_nulls_count', - ' FROM ({{=it._query}}) __cdb_histogram_nulls', - ' WHERE {{=it._column}} IS NULL', - ')' -].join('\n')); +const nullsQueryTpl = ctx => ` + __cdb_nulls AS ( + SELECT + count(*) AS __cdb_nulls_count + FROM (${it._query}) __cdb_histogram_nulls + WHERE ${it._column} IS NULL + ) +`; var infinitiesQueryTpl = dot.template([ '__cdb_infinities AS (', From 7ea6b3e37181ecbbed581b28d1446bba95859068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 12:21:30 +0200 Subject: [PATCH 009/177] Use template string for infinitiesQueryTpl --- lib/cartodb/models/dataview/histogram.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 35c9b43c..af18ea12 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -133,17 +133,17 @@ const nullsQueryTpl = ctx => ` ) `; -var infinitiesQueryTpl = dot.template([ - '__cdb_infinities AS (', - ' SELECT', - ' count(*) AS __cdb_infinities_count', - ' FROM ({{=it._query}}) __cdb_infinities_query', - ' WHERE', - ' {{=it._column}} = \'infinity\'::float', - ' OR', - ' {{=it._column}} = \'-infinity\'::float', - ')' -].join('\n')); +const infinitiesQueryTpl = ctx => ` + __cdb_infinities AS ( + SELECT + count(*) AS __cdb_infinities_count + FROM (${ctx._query}) __cdb_infinities_query + WHERE + ${ctx._column} = 'infinity'::float + OR + ${ctx._column} = '-infinity'::float + ) +`; var nansQueryTpl = dot.template([ '__cdb_nans AS (', From 734cfa6d8392ffa2a3e1697119136a434734eecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 15:51:31 +0200 Subject: [PATCH 010/177] Fix undefined argument --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index af18ea12..fdacbbe0 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -128,8 +128,8 @@ const nullsQueryTpl = ctx => ` __cdb_nulls AS ( SELECT count(*) AS __cdb_nulls_count - FROM (${it._query}) __cdb_histogram_nulls - WHERE ${it._column} IS NULL + FROM (${ctx._query}) __cdb_histogram_nulls + WHERE ${ctx._column} IS NULL ) `; From adcff54589dd3c3af8b7e7a5de94ee75e841c985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 15:53:06 +0200 Subject: [PATCH 011/177] Use template string for nansQueryTpl --- lib/cartodb/models/dataview/histogram.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index fdacbbe0..c9d89fd5 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -145,14 +145,14 @@ const infinitiesQueryTpl = ctx => ` ) `; -var nansQueryTpl = dot.template([ - '__cdb_nans AS (', - ' SELECT', - ' count(*) AS __cdb_nans_count', - ' FROM ({{=it._query}}) __cdb_nans_query', - ' WHERE {{=it._column}} = \'NaN\'::float', - ')' -].join('\n')); +const nansQueryTpl = ctx => ` + __cdb_nans AS ( + SELECT + count(*) AS __cdb_nans_count + FROM (${ctx._query}) __cdb_nans_query + WHERE ${ctx._column} = 'NaN'::float + ) +`; var histogramQueryTpl = dot.template([ 'SELECT', From 27393641935ccced2e968820b7fed77225dbc450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 5 Sep 2017 15:53:21 +0200 Subject: [PATCH 012/177] Use template string for histogramQueryTpl --- lib/cartodb/models/dataview/histogram.js | 66 ++++++++++++++---------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c9d89fd5..7a391ecb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -154,34 +154,44 @@ const nansQueryTpl = ctx => ` ) `; -var histogramQueryTpl = dot.template([ - 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', - ' __cdb_bins_number AS bins_number,', - ' __cdb_nulls_count AS nulls_count,', - ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', - ' __cdb_nans_count AS nans_count,{{?}}', - ' __cdb_avg_val AS avg_val,', - ' CASE WHEN __cdb_min_val = __cdb_max_val', - ' THEN 0', - ' ELSE GREATEST(', - ' 1,', - ' LEAST(', - ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', - ' __cdb_bins_number', - ' )', - ' ) - 1', - ' END AS bin,', - ' min({{=it._column}})::numeric AS min,', - ' max({{=it._column}})::numeric AS max,', - ' avg({{=it._column}})::numeric AS avg,', - ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', - ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', - 'GROUP BY bin, bins_number, bin_width, nulls_count,', - ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', - 'ORDER BY bin' -].join('\n')); +const specialNumericValuesColumnDefinitionTpl = () => ` + __cdb_infinities_count AS infinities_count, + __cdb_nans_count AS nans_count +`; + +const specialNumericValuesCTETpl = () => ` + __cdb_infinities, __cdb_nans +`; + +const specialNumericValuesColumnTpl = () => ` + infinities_count, nans_count +`; + +const histogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + ${ctx._isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} + __cdb_avg_val AS avg_val, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET(${ctx._column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min(${ctx._column})::numeric AS min, + max(${ctx._column})::numeric AS max, + avg(${ctx._column})::numeric AS avg, + count(*) AS freq + FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins ${ctx._isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} + GROUP BY bin, bins_number, bin_width, nulls_count, avg_val${ctx._isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} + ORDER BY bin +`; var dateBasicsQueryTpl = dot.template([ '__cdb_basics AS (', From 0b79ac76dbd3566604c945c376a8e397c547b56e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 10:39:17 +0200 Subject: [PATCH 013/177] Use template string for dateBasicsQueryTpl --- lib/cartodb/models/dataview/histogram.js | 30 +++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7a391ecb..2adcc944 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -193,20 +193,22 @@ const histogramQueryTpl = ctx => ` ORDER BY bin `; -var dateBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', - ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', - ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )) AS __cdb_start_date,', - ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', - ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) __cdb_basics_query', - ')' -].join(' \n')); +const dateBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(date_part('epoch', ${ctx._column})) AS __cdb_max_val, + min(date_part('epoch', ${ctx._column})) AS __cdb_min_val, + avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' + ) + ) AS __cdb_start_date, + max(${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}') AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx._query}) __cdb_basics_query + ) +`; var dateOverrideBasicsQueryTpl = dot.template([ '__cdb_basics AS (', From 0ede3013dbf1ef60b9ba2dcf8b87fe8a983b928a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 10:57:38 +0200 Subject: [PATCH 014/177] Use template string for dateOverrideBasicsQueryTpl --- lib/cartodb/models/dataview/histogram.js | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 2adcc944..9ec42906 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -210,25 +210,25 @@ const dateBasicsQueryTpl = ctx => ` ) `; -var dateOverrideBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._end}}) AS __cdb_max_val,', - ' min({{=it._start}}) AS __cdb_min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', - ' min(', - ' date_trunc(', - ' \'{{=it._aggregation}}\',', - ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )', - ' ) AS __cdb_start_date,', - ' max(', - ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AS __cdb_end_date,', - ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) __cdb_basics_query', - ')' -].join(' \n')); +const dateOverrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx._end}) AS __cdb_max_val, + min(${ctx._start}) AS __cdb_min_val, + avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx._aggregation}', + TO_TIMESTAMP(${ctx._start})::timestamp AT TIME ZONE '${ctx._offset}' + ) + ) AS __cdb_start_date, + max( + TO_TIMESTAMP(${ctx._end})::timestamp AT TIME ZONE '${ctx._offset}' + ) AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx._query}) __cdb_basics_query + ) +`; var dateBinsQueryTpl = dot.template([ '__cdb_bins AS (', From 4f75f6c07bf34ae7df8af5370ca34c30f5058527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 11:41:24 +0200 Subject: [PATCH 015/177] Use template string for dateBinsQueryTpl and dateHistogramQueryTpl --- lib/cartodb/models/dataview/histogram.js | 105 ++++++++++++----------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 9ec42906..42dd15d3 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -230,58 +230,61 @@ const dateOverrideBasicsQueryTpl = ctx => ` ) `; -var dateBinsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT', - ' __cdb_bins_array,', - ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', - ' FROM (', - ' SELECT', - ' ARRAY(', - ' SELECT GENERATE_SERIES(', - ' __cdb_start_date::timestamptz,', - ' __cdb_end_date::timestamptz,', - ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', - ' )', - ' ) AS __cdb_bins_array', - ' FROM __cdb_basics', - ' ) __cdb_bins_array_query', - ')' -].join('\n')); +const dateBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + __cdb_bins_array, + ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number + FROM ( + SELECT + ARRAY( + SELECT GENERATE_SERIES( + __cdb_start_date::timestamptz, + __cdb_end_date::timestamptz, + ${ctx._aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx._aggregation}'::interval`} + ) + ) AS __cdb_bins_array + FROM __cdb_basics + ) __cdb_bins_array_query + ) +`; -var dateHistogramQueryTpl = dot.template([ - 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', - ' __cdb_bins_number AS bins_number,', - ' __cdb_nulls_count AS nulls_count,', - ' CASE WHEN __cdb_min_val = __cdb_max_val', - ' THEN 0', - ' ELSE GREATEST(1, LEAST(', - ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', - ' __cdb_bins_array', - ' ),', - ' __cdb_bins_number', - ' )) - 1', - ' END AS bin,', - ' min(', - ' date_part(', - ' \'epoch\', ', - ' date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AT TIME ZONE \'{{=it._offset}}\'', - ' )', - ' )::numeric AS timestamp,', - ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', - ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', - ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', - ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', - ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', - 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', - 'ORDER BY bin' -].join('\n')); +const dateHistogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET( + ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}', + __cdb_bins_array + ), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min( + date_part( + 'epoch', + date_trunc( + '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' + ) AT TIME ZONE '${ctx._offset}' + ) + )::numeric AS timestamp, + date_part('epoch', __cdb_start_date)::numeric AS timestamp_start, + min(date_part('epoch', ${ctx._column}))::numeric AS min, + max(date_part('epoch', ${ctx._column}))::numeric AS max, + avg(date_part('epoch', ${ctx._column}))::numeric AS avg, + count(*) AS freq + FROM (${ctx._query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls + WHERE date_part('epoch', ${ctx._column}) IS NOT NULL + GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start + ORDER BY bin +`; var TYPE = 'histogram'; From 8e2535745eaa034dce3914755f12379afd709479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 11:44:25 +0200 Subject: [PATCH 016/177] Use template string for columnCastTpl --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 42dd15d3..4cbfd0f1 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -5,7 +5,7 @@ var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var columnCastTpl = ctx => `date_part('epoch', ${ctx.column})` const dateIntervalQueryTpl = ctx => ` WITH From bbd42b73f2bc9e56495e26fd9150f1826d342c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 11:44:52 +0200 Subject: [PATCH 017/177] Remove dot requirement --- lib/cartodb/models/dataview/histogram.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4cbfd0f1..b2782e31 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -2,10 +2,7 @@ var _ = require('underscore'); var BaseWidget = require('./base'); var debug = require('debug')('windshaft:dataview:histogram'); -var dot = require('dot'); -dot.templateSettings.strip = false; - -var columnCastTpl = ctx => `date_part('epoch', ${ctx.column})` +const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})` const dateIntervalQueryTpl = ctx => ` WITH From af4b3d81cd1a881d0cb10eb423785ac43ffe0da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 11:55:40 +0200 Subject: [PATCH 018/177] make happy to jshint --- lib/cartodb/models/dataview/histogram.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b2782e31..0c57a563 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -2,7 +2,7 @@ var _ = require('underscore'); var BaseWidget = require('./base'); var debug = require('debug')('windshaft:dataview:histogram'); -const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})` +const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; const dateIntervalQueryTpl = ctx => ` WITH @@ -185,8 +185,10 @@ const histogramQueryTpl = ctx => ` max(${ctx._column})::numeric AS max, avg(${ctx._column})::numeric AS avg, count(*) AS freq - FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins ${ctx._isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} - GROUP BY bin, bins_number, bin_width, nulls_count, avg_val${ctx._isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} + FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins + ${ctx._isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} + GROUP BY bin, bins_number, bin_width, nulls_count, avg_val + ${ctx._isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} ORDER BY bin `; From 5ba2dfbbd6cb99bdd972c54f1fdd236c8235ce6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 13:43:54 +0200 Subject: [PATCH 019/177] Use ES6 class syntax --- lib/cartodb/models/dataview/histogram.js | 616 +++++++++++------------ 1 file changed, 308 insertions(+), 308 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 0c57a563..25a08da0 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -307,159 +307,349 @@ Time series: } } */ -function Histogram(query, options, queries) { - if (!_.isString(options.column)) { - throw new Error('Histogram expects `column` in widget options'); + +module.exports = class Histogram extends BaseWidget { + constructor (query, options, queries) { + super(); + + if (!_.isString(options.column)) { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.bins = options.bins; + this.aggregation = options.aggregation; + this.offset = options.offset; + + this._columnType = null; } - this.query = query; - this.queries = queries; - this.column = options.column; - this.bins = options.bins; - this.aggregation = options.aggregation; - this.offset = options.offset; + sql (psql, override, callback) { + var self = this; - this._columnType = null; -} + if (!callback) { + callback = override; + override = {}; + } -Histogram.prototype = new BaseWidget(); -Histogram.prototype.constructor = Histogram; + if (this._columnType === null) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + // assume numeric, will fail later + self._columnType = 'numeric'; + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); + } + self.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } -module.exports = Histogram; - -Histogram.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; + this._buildQuery(psql, override, callback); } - if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { - // assume numeric, will fail later - self._columnType = 'numeric'; - if (!err && !!type) { - self._columnType = Object.keys(type).find(function (key) { - return type[key]; - }); - } - self.sql(psql, override, callback); - }, true); // use read-only transaction - return null; + isDateHistogram (override) { + return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); } - this._buildQuery(psql, override, callback); -}; + _buildQuery (psql, override, callback) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; + var _query = this.query; -Histogram.prototype.isDateHistogram = function (override) { - return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); -}; + if (this.isDateHistogram(override)) { + return this._buildDateHistogramQuery(psql, override, callback); + } -Histogram.prototype._buildQuery = function (psql, override, callback) { - var filteredQuery, basicsQuery, binsQuery; - var _column = this.column; - var _query = this.query; + if (this._columnType === 'date') { + _column = columnCastTpl({column: _column}); + } - if (this.isDateHistogram(override)) { - return this._buildDateHistogramQuery(psql, override, callback); - } - - if (this._columnType === 'date') { - _column = columnCastTpl({column: _column}); - } - - filteredQuery = filteredQueryTpl({ - _isFloatColumn: this._columnType === 'float', - _query: _query, - _column: _column - }); - - if (this._shouldOverride(override)) { - debug('overriding with %j', override); - basicsQuery = overrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _start: getBinStart(override), - _end: getBinEnd(override) - }); - - binsQuery = [ - overrideBinsQueryTpl({ - _bins: override.bins - }) - ].join(',\n'); - } else { - basicsQuery = basicsQueryTpl({ + filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }); - if (this._shouldOverrideBins(override)) { + if (this._shouldOverride(override)) { + debug('overriding with %j', override); + basicsQuery = overrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _start: getBinStart(override), + _end: getBinEnd(override) + }); + binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins }) ].join(',\n'); } else { - binsQuery = [ - iqrQueryTpl({ - _query: _query, - _column: _column - }), - binsQueryTpl({ - _query: _query, - _minBins: BIN_MIN_NUMBER, - _maxBins: BIN_MAX_NUMBER - }) - ].join(',\n'); - } - } - - var cteSql = [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }) - ]; - - if (this._columnType === 'float') { - cteSql.push( - infinitiesQueryTpl({ + basicsQuery = basicsQueryTpl({ _query: _query, _column: _column - }), - nansQueryTpl({ + }); + + if (this._shouldOverrideBins(override)) { + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + binsQuery = [ + iqrQueryTpl({ + _query: _query, + _column: _column + }), + binsQueryTpl({ + _query: _query, + _minBins: BIN_MIN_NUMBER, + _maxBins: BIN_MAX_NUMBER + }) + ].join(',\n'); + } + } + + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ _query: _query, _column: _column }) - ); + ]; + + if (this._columnType === 'float') { + cteSql.push( + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ + _query: _query, + _column: _column + }) + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), + histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', + _query: _query, + _column: _column + }) + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); } - var histogramSql = [ - "WITH", - cteSql.join(',\n'), - histogramQueryTpl({ - _isFloatColumn: this._columnType === 'float', + _shouldOverride (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); + } + + _shouldOverrideBins (override) { + return override && _.has(override, 'bins'); + } + + _buildDateHistogramQuery (psql, override, callback) { + var _column = this.column; + var _query = this.query; + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + + if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + + if (_aggregation === 'auto') { + this.getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + override.aggregation = this.aggregation; + this._buildDateHistogramQuery(psql, override, callback); + }.bind(this)); + return null; + } + + var dateBasicsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end')) { + dateBasicsQuery = dateOverrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _start: getBinStart(override), + _end: getBinEnd(override), + _offset: parseOffset(_offset, _aggregation) + }); + } else { + dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); + } + + var dateBinsQuery = [ + dateBinsQueryTpl({ + _aggregation: _aggregation + }) + ].join(',\n'); + + var nullsQuery = nullsQueryTpl({ _query: _query, _column: _column - }) - ].join('\n'); + }); - debug(histogramSql); + var dateHistogramQuery = dateHistogramQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); - return callback(null, histogramSql); -}; + var histogramSql = [ + "WITH", + [ + dateBasicsQuery, + dateBinsQuery, + nullsQuery + ].join(',\n'), + dateHistogramQuery + ].join('\n'); -Histogram.prototype._shouldOverride = function (override) { - return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); -}; + debug(histogramSql); -Histogram.prototype._shouldOverrideBins = function (override) { - return override && _.has(override, 'bins'); + return callback(null, histogramSql); + } + + getAutomaticAggregation (psql, callback) { + var dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + debug(dateIntervalQuery); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + var aggegations = result.rows[0]; + var aggregation = Object.keys(aggegations) + .map(function (key) { + return { + name: key, + value: aggegations[key] + }; + }) + .reduce(function (closer, current) { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + var closerDiff = MAX_INTERVAL_VALUE - closer.value; + var currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); + } + + format (result, override) { + override = override || {}; + var buckets = []; + + var binsCount = getBinsCount(override); + var width = getWidth(override); + var binsStart = getBinStart(override); + var nulls = 0; + var infinities = 0; + var nans = 0; + var avg; + var timestampStart; + var aggregation; + var offset; + + if (result.rows.length) { + var firstRow = result.rows[0]; + binsCount = firstRow.bins_number; + width = firstRow.bin_width || width; + avg = firstRow.avg_val; + nulls = firstRow.nulls_count; + timestampStart = firstRow.timestamp_start; + infinities = firstRow.infinities_count; + nans = firstRow.nans_count; + binsStart = populateBinStart(override, firstRow); + + if (Number.isFinite(timestampStart)) { + aggregation = getAggregation(override, this.aggregation); + offset = getOffset(override, this.offset); + } + + buckets = result.rows.map(function(row) { + return _.omit( + row, + 'bins_number', + 'bin_width', + 'nulls_count', + 'infinities_count', + 'nans_count', + 'avg_val', + 'timestamp_start' + ); + }); + } + + return { + aggregation: aggregation, + offset: offset, + timestamp_start: timestampStart, + bin_width: width, + bins_count: binsCount, + bins_start: binsStart, + nulls: nulls, + infinities: infinities, + nans: nans, + avg: avg, + bins: buckets + }; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); + } }; var DATE_AGGREGATIONS = { @@ -473,184 +663,6 @@ var DATE_AGGREGATIONS = { 'year': true }; -Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { - var _column = this.column; - var _query = this.query; - var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; - - if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { - return callback(new Error('Invalid aggregation value. Valid ones: ' + - Object.keys(DATE_AGGREGATIONS).join(', ') - )); - } - - if (_aggregation === 'auto') { - this.getAutomaticAggregation(psql, function (err, aggregation) { - if (err || aggregation === 'none') { - this.aggregation = 'day'; - } else { - this.aggregation = aggregation; - } - override.aggregation = this.aggregation; - this._buildDateHistogramQuery(psql, override, callback); - }.bind(this)); - return null; - } - - var dateBasicsQuery; - - if (override && _.has(override, 'start') && _.has(override, 'end')) { - dateBasicsQuery = dateOverrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _start: getBinStart(override), - _end: getBinEnd(override), - _offset: parseOffset(_offset, _aggregation) - }); - } else { - dateBasicsQuery = dateBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) - }); - } - - var dateBinsQuery = [ - dateBinsQueryTpl({ - _aggregation: _aggregation - }) - ].join(',\n'); - - var nullsQuery = nullsQueryTpl({ - _query: _query, - _column: _column - }); - - var dateHistogramQuery = dateHistogramQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) - }); - - var histogramSql = [ - "WITH", - [ - dateBasicsQuery, - dateBinsQuery, - nullsQuery - ].join(',\n'), - dateHistogramQuery - ].join('\n'); - - debug(histogramSql); - - return callback(null, histogramSql); -}; - -Histogram.prototype.getAutomaticAggregation = function (psql, callback) { - var dateIntervalQuery = dateIntervalQueryTpl({ - query: this.query, - column: this.column - }); - - debug(dateIntervalQuery); - - psql.query(dateIntervalQuery, function (err, result) { - if (err) { - return callback(err); - } - - var aggegations = result.rows[0]; - var aggregation = Object.keys(aggegations) - .map(function (key) { - return { - name: key, - value: aggegations[key] - }; - }) - .reduce(function (closer, current) { - if (current.value > MAX_INTERVAL_VALUE) { - return closer; - } - - var closerDiff = MAX_INTERVAL_VALUE - closer.value; - var currentDiff = MAX_INTERVAL_VALUE - current.value; - - if (Number.isFinite(current.value) && closerDiff > currentDiff) { - return current; - } - - return closer; - }, { name: 'none', value: -1 }); - - callback(null, aggregation.name); - }); -}; - -Histogram.prototype.format = function(result, override) { - override = override || {}; - var buckets = []; - - var binsCount = getBinsCount(override); - var width = getWidth(override); - var binsStart = getBinStart(override); - var nulls = 0; - var infinities = 0; - var nans = 0; - var avg; - var timestampStart; - var aggregation; - var offset; - - if (result.rows.length) { - var firstRow = result.rows[0]; - binsCount = firstRow.bins_number; - width = firstRow.bin_width || width; - avg = firstRow.avg_val; - nulls = firstRow.nulls_count; - timestampStart = firstRow.timestamp_start; - infinities = firstRow.infinities_count; - nans = firstRow.nans_count; - binsStart = populateBinStart(override, firstRow); - - if (Number.isFinite(timestampStart)) { - aggregation = getAggregation(override, this.aggregation); - offset = getOffset(override, this.offset); - } - - buckets = result.rows.map(function(row) { - return _.omit( - row, - 'bins_number', - 'bin_width', - 'nulls_count', - 'infinities_count', - 'nans_count', - 'avg_val', - 'timestamp_start' - ); - }); - } - - return { - aggregation: aggregation, - offset: offset, - timestamp_start: timestampStart, - bin_width: width, - bins_count: binsCount, - bins_start: binsStart, - nulls: nulls, - infinities: infinities, - nans: nans, - avg: avg, - bins: buckets - }; -}; - function getAggregation(override, aggregation) { return override && override.aggregation ? override.aggregation : aggregation; } @@ -720,15 +732,3 @@ function populateBinStart(override, firstRow) { return binStart; } - -Histogram.prototype.getType = function() { - return TYPE; -}; - -Histogram.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _column: this.column, - _query: this.query - }); -}; From 1be66e155253fb83a2b5ef5e7b9af88917a71564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 13:47:19 +0200 Subject: [PATCH 020/177] Use const for requirements --- lib/cartodb/models/dataview/histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 25a08da0..5a4f713b 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,6 +1,6 @@ -var _ = require('underscore'); -var BaseWidget = require('./base'); -var debug = require('debug')('windshaft:dataview:histogram'); +const _ = require('underscore'); +const BaseWidget = require('./base'); +const debug = require('debug')('windshaft:dataview:histogram'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; From 96740b82ed3a84d8066cc6bc3f222474601d5eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 13:52:02 +0200 Subject: [PATCH 021/177] Move getAggregation function to a class method --- lib/cartodb/models/dataview/histogram.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5a4f713b..dbe8f014 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -606,7 +606,7 @@ module.exports = class Histogram extends BaseWidget { binsStart = populateBinStart(override, firstRow); if (Number.isFinite(timestampStart)) { - aggregation = getAggregation(override, this.aggregation); + aggregation = this.getAggregation(override); offset = getOffset(override, this.offset); } @@ -650,6 +650,10 @@ module.exports = class Histogram extends BaseWidget { _query: this.query }); } + + getAggregation (override) { + return override && override.aggregation ? override.aggregation : this.aggregation; + } }; var DATE_AGGREGATIONS = { @@ -663,9 +667,6 @@ var DATE_AGGREGATIONS = { 'year': true }; -function getAggregation(override, aggregation) { - return override && override.aggregation ? override.aggregation : aggregation; -} function getOffset(override, offset) { if (override && override.offset) { From 288656301bd4b853e53521a6cda493ab985b01fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 13:53:48 +0200 Subject: [PATCH 022/177] Move getOffset function to a class method --- lib/cartodb/models/dataview/histogram.js | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dbe8f014..0e6279b8 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -607,7 +607,7 @@ module.exports = class Histogram extends BaseWidget { if (Number.isFinite(timestampStart)) { aggregation = this.getAggregation(override); - offset = getOffset(override, this.offset); + offset = this.getOffset(override); } buckets = result.rows.map(function(row) { @@ -654,6 +654,17 @@ module.exports = class Histogram extends BaseWidget { getAggregation (override) { return override && override.aggregation ? override.aggregation : this.aggregation; } + + getOffset (override) { + if (override && override.offset) { + return override.offset; + } + if (this.offset) { + return this.offset; + } + + return 0; + } }; var DATE_AGGREGATIONS = { @@ -667,18 +678,6 @@ var DATE_AGGREGATIONS = { 'year': true }; - -function getOffset(override, offset) { - if (override && override.offset) { - return override.offset; - } - if (offset) { - return offset; - } - - return 0; -} - function getBinStart(override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); From 392e004879019318c5878ac0ba0f6b08cd573ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 15:38:23 +0200 Subject: [PATCH 023/177] Move getBinStart and populateBinStart function to a class method --- lib/cartodb/models/dataview/histogram.js | 49 ++++++++++++------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 0e6279b8..6a5081c1 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -379,7 +379,7 @@ module.exports = class Histogram extends BaseWidget { basicsQuery = overrideBasicsQueryTpl({ _query: _query, _column: _column, - _start: getBinStart(override), + _start: this.getBinStart(override), _end: getBinEnd(override) }); @@ -493,7 +493,7 @@ module.exports = class Histogram extends BaseWidget { _query: _query, _column: _column, _aggregation: _aggregation, - _start: getBinStart(override), + _start: this.getBinStart(override), _end: getBinEnd(override), _offset: parseOffset(_offset, _aggregation) }); @@ -585,7 +585,7 @@ module.exports = class Histogram extends BaseWidget { var binsCount = getBinsCount(override); var width = getWidth(override); - var binsStart = getBinStart(override); + var binsStart = this.getBinStart(override); var nulls = 0; var infinities = 0; var nans = 0; @@ -603,7 +603,7 @@ module.exports = class Histogram extends BaseWidget { timestampStart = firstRow.timestamp_start; infinities = firstRow.infinities_count; nans = firstRow.nans_count; - binsStart = populateBinStart(override, firstRow); + binsStart = this.populateBinStart(override, firstRow); if (Number.isFinite(timestampStart)) { aggregation = this.getAggregation(override); @@ -665,6 +665,27 @@ module.exports = class Histogram extends BaseWidget { return 0; } + + getBinStart (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.min(override.start, override.end); + } + return override.start || 0; + } + + populateBinStart (override, firstRow) { + var binStart; + + if (firstRow.hasOwnProperty('timestamp')) { + binStart = firstRow.timestamp; + } else if (override.hasOwnProperty('start')) { + binStart = this.getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; + } }; var DATE_AGGREGATIONS = { @@ -678,13 +699,6 @@ var DATE_AGGREGATIONS = { 'year': true }; -function getBinStart(override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.min(override.start, override.end); - } - return override.start || 0; -} - function getBinEnd(override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.max(override.start, override.end); @@ -719,16 +733,3 @@ function parseOffset(offset, aggregation) { return '' + offsetInHours; } -function populateBinStart(override, firstRow) { - var binStart; - - if (firstRow.hasOwnProperty('timestamp')) { - binStart = firstRow.timestamp; - } else if (override.hasOwnProperty('start')) { - binStart = getBinStart(override); - } else { - binStart = firstRow.min; - } - - return binStart; -} From 49a60caffcf6d94b615211cd32e5263bf937b890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 15:42:29 +0200 Subject: [PATCH 024/177] Move getBinEnd function to a class method --- lib/cartodb/models/dataview/histogram.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6a5081c1..01b62148 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -380,7 +380,7 @@ module.exports = class Histogram extends BaseWidget { _query: _query, _column: _column, _start: this.getBinStart(override), - _end: getBinEnd(override) + _end: this.getBinEnd(override) }); binsQuery = [ @@ -494,7 +494,7 @@ module.exports = class Histogram extends BaseWidget { _column: _column, _aggregation: _aggregation, _start: this.getBinStart(override), - _end: getBinEnd(override), + _end: this.getBinEnd(override), _offset: parseOffset(_offset, _aggregation) }); } else { @@ -686,6 +686,14 @@ module.exports = class Histogram extends BaseWidget { return binStart; } + + getBinEnd (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.max(override.start, override.end); + } + return override.end || 0; + } + }; var DATE_AGGREGATIONS = { @@ -699,13 +707,6 @@ var DATE_AGGREGATIONS = { 'year': true }; -function getBinEnd(override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.max(override.start, override.end); - } - return override.end || 0; -} - function getBinsCount(override) { return override.bins || 0; } From e84f30488fb14f97d2b1697578938b09a770e523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 15:45:51 +0200 Subject: [PATCH 025/177] Move getBinsCount function to a class method --- lib/cartodb/models/dataview/histogram.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 01b62148..c2df2c5f 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -583,7 +583,7 @@ module.exports = class Histogram extends BaseWidget { override = override || {}; var buckets = []; - var binsCount = getBinsCount(override); + var binsCount = this.getBinsCount(override); var width = getWidth(override); var binsStart = this.getBinStart(override); var nulls = 0; @@ -694,6 +694,9 @@ module.exports = class Histogram extends BaseWidget { return override.end || 0; } + getBinsCount (override) { + return override.bins || 0; + } }; var DATE_AGGREGATIONS = { @@ -707,10 +710,6 @@ var DATE_AGGREGATIONS = { 'year': true }; -function getBinsCount(override) { - return override.bins || 0; -} - function getWidth(override) { var width = 0; var binsCount = override.bins; From 9e3e1cad9a64ee346cee0c7cf9c239c0d5ca1088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 15:47:21 +0200 Subject: [PATCH 026/177] Move getWidth function to a class method --- lib/cartodb/models/dataview/histogram.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c2df2c5f..3e9868d9 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -584,7 +584,7 @@ module.exports = class Histogram extends BaseWidget { var buckets = []; var binsCount = this.getBinsCount(override); - var width = getWidth(override); + var width = this.getWidth(override); var binsStart = this.getBinStart(override); var nulls = 0; var infinities = 0; @@ -697,6 +697,17 @@ module.exports = class Histogram extends BaseWidget { getBinsCount (override) { return override.bins || 0; } + + getWidth (override) { + var width = 0; + var binsCount = override.bins; + + if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { + width = (override.end - override.start) / binsCount; + } + + return width; + } }; var DATE_AGGREGATIONS = { @@ -710,16 +721,6 @@ var DATE_AGGREGATIONS = { 'year': true }; -function getWidth(override) { - var width = 0; - var binsCount = override.bins; - - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; - } - - return width; -} function parseOffset(offset, aggregation) { if (!offset) { From c37ef36a611b53192067f03b94a2c116ad9be292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 15:49:05 +0200 Subject: [PATCH 027/177] Move parseOffset function to a class method --- lib/cartodb/models/dataview/histogram.js | 32 +++++++++++------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 3e9868d9..9ab349df 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -495,14 +495,14 @@ module.exports = class Histogram extends BaseWidget { _aggregation: _aggregation, _start: this.getBinStart(override), _end: this.getBinEnd(override), - _offset: parseOffset(_offset, _aggregation) + _offset: this.parseOffset(_offset, _aggregation) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) + _offset: this.parseOffset(_offset, _aggregation) }); } @@ -521,7 +521,7 @@ module.exports = class Histogram extends BaseWidget { _query: _query, _column: _column, _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) + _offset: this.parseOffset(_offset, _aggregation) }); var histogramSql = [ @@ -708,6 +708,18 @@ module.exports = class Histogram extends BaseWidget { return width; } + + parseOffset (offset, aggregation) { + if (!offset) { + return '0'; + } + if (aggregation === 'hour' || aggregation === 'minute') { + return '0'; + } + + var offsetInHours = Math.ceil(offset / 3600); + return '' + offsetInHours; + } }; var DATE_AGGREGATIONS = { @@ -720,17 +732,3 @@ var DATE_AGGREGATIONS = { 'quarter': true, 'year': true }; - - -function parseOffset(offset, aggregation) { - if (!offset) { - return '0'; - } - if (aggregation === 'hour' || aggregation === 'minute') { - return '0'; - } - - var offsetInHours = Math.ceil(offset / 3600); - return '' + offsetInHours; -} - From da1d0550f6b48edaaff2751aa82157a7a8e53ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 15:52:13 +0200 Subject: [PATCH 028/177] Use const keyword for constants --- lib/cartodb/models/dataview/histogram.js | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 9ab349df..8726268d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -44,10 +44,6 @@ const dateIntervalQueryTpl = ctx => ` FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds `; -var MAX_INTERVAL_VALUE = 366; -var BIN_MIN_NUMBER = 6; -var BIN_MAX_NUMBER = 48; - const filterOutSpecialNumericValues = ctx => ` ${ctx._column} != 'infinity'::float AND @@ -285,7 +281,22 @@ const dateHistogramQueryTpl = ctx => ` ORDER BY bin `; -var TYPE = 'histogram'; +const MAX_INTERVAL_VALUE = 366; +const BIN_MIN_NUMBER = 6; +const BIN_MAX_NUMBER = 48; + +const TYPE = 'histogram'; + +const DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; /** Numeric histogram: @@ -721,14 +732,3 @@ module.exports = class Histogram extends BaseWidget { return '' + offsetInHours; } }; - -var DATE_AGGREGATIONS = { - 'auto': true, - 'minute': true, - 'hour': true, - 'day': true, - 'week': true, - 'month': true, - 'quarter': true, - 'year': true -}; From 3e55bd2abb80f4bb53f2e562ba66c4f1eb9f3c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 15:56:52 +0200 Subject: [PATCH 029/177] Make happy to jshint --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 8726268d..98d59214 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -731,4 +731,4 @@ module.exports = class Histogram extends BaseWidget { var offsetInHours = Math.ceil(offset / 3600); return '' + offsetInHours; } -}; +} From 6e0dc8666d103c3e7d7f3bcc7d6a3bf753e9b973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 16:14:29 +0200 Subject: [PATCH 030/177] Use .hasOwnProperty() instead of underscore's equivalent --- lib/cartodb/models/dataview/histogram.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 98d59214..a1e08494 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -465,11 +465,14 @@ module.exports = class Histogram extends BaseWidget { } _shouldOverride (override) { - return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); + return override && + override.hasOwnProperty('start') && + override.hasOwnProperty('end') && + override.hasOwnProperty('bins'); } _shouldOverrideBins (override) { - return override && _.has(override, 'bins'); + return override && override.hasOwnProperty('bins'); } _buildDateHistogramQuery (psql, override, callback) { @@ -499,7 +502,7 @@ module.exports = class Histogram extends BaseWidget { var dateBasicsQuery; - if (override && _.has(override, 'start') && _.has(override, 'end')) { + if (override && override.hasOwnProperty('start') && override.hasOwnProperty('end')) { dateBasicsQuery = dateOverrideBasicsQueryTpl({ _query: _query, _column: _column, @@ -731,4 +734,4 @@ module.exports = class Histogram extends BaseWidget { var offsetInHours = Math.ceil(offset / 3600); return '' + offsetInHours; } -} +}; From 8ac041805c51ec806759d1bc05ee62cb5438ab2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 16:20:01 +0200 Subject: [PATCH 031/177] Use typeof !== string instead of underscore's equivalent --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index a1e08494..8624efc5 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -323,7 +323,7 @@ module.exports = class Histogram extends BaseWidget { constructor (query, options, queries) { super(); - if (!_.isString(options.column)) { + if (typeof options.column !== 'string') { throw new Error('Histogram expects `column` in widget options'); } From b5d2de8edc35811f4cc207ccfbb4bd7ee1c0bfc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 16:49:00 +0200 Subject: [PATCH 032/177] Do not use _.omit() --- lib/cartodb/models/dataview/histogram.js | 29 ++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 8624efc5..3d758b17 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -624,18 +624,23 @@ module.exports = class Histogram extends BaseWidget { offset = this.getOffset(override); } - buckets = result.rows.map(function(row) { - return _.omit( - row, - 'bins_number', - 'bin_width', - 'nulls_count', - 'infinities_count', - 'nans_count', - 'avg_val', - 'timestamp_start' - ); - }); + // TODO: after applying strategy we could use specialized .format() + // in order to know what props we want to expose using destructuring assignment + + const blacklisted = [ + 'bins_number', + 'bin_width', + 'nulls_count', + 'infinities_count', + 'nans_count', + 'avg_val', + 'timestamp_start' + ]; + + buckets = result.rows.map(row => Object.keys(row) + .filter((key) => blacklisted.indexOf(key) < 0) + .reduce((newObj, key) => Object.assign(newObj, { [key]: row[key] }), {}) + ); } return { From 4edf18f77a733d125e4688b3cbbcb01bd4531c8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 16:52:17 +0200 Subject: [PATCH 033/177] Remove underscore requirement --- lib/cartodb/models/dataview/histogram.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 3d758b17..271c3e95 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,4 +1,3 @@ -const _ = require('underscore'); const BaseWidget = require('./base'); const debug = require('debug')('windshaft:dataview:histogram'); From 9f52e58be896bcc670da3951d81b57e4479c0249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 16:54:08 +0200 Subject: [PATCH 034/177] Rename BaseWidget by BaseDataview --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 271c3e95..b732f712 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,4 +1,4 @@ -const BaseWidget = require('./base'); +const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:histogram'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; @@ -318,7 +318,7 @@ Time series: } */ -module.exports = class Histogram extends BaseWidget { +module.exports = class Histogram extends BaseDataview { constructor (query, options, queries) { super(); From 1ac8455dc28cc8f66c1f207f5f812f371dc3aeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 18:13:34 +0200 Subject: [PATCH 035/177] Use template strings to build histogram query --- lib/cartodb/models/dataview/histogram.js | 110 ++++++----------------- 1 file changed, 26 insertions(+), 84 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b732f712..60d6cd36 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -366,103 +366,45 @@ module.exports = class Histogram extends BaseDataview { } _buildQuery (psql, override, callback) { - var filteredQuery, basicsQuery, binsQuery; - var _column = this.column; - var _query = this.query; if (this.isDateHistogram(override)) { return this._buildDateHistogramQuery(psql, override, callback); } - if (this._columnType === 'date') { - _column = columnCastTpl({column: _column}); - } - - filteredQuery = filteredQueryTpl({ + const histogramSql = this.buildNumericHistogramQueryTpl({ + _override: override, + _column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, _isFloatColumn: this._columnType === 'float', - _query: _query, - _column: _column + _query: this.query, + _start: this.getBinStart(override), + _end: this.getBinEnd(override), + _minBins: BIN_MIN_NUMBER, + _maxBins: BIN_MAX_NUMBER, + _bins: override.bins }); - if (this._shouldOverride(override)) { - debug('overriding with %j', override); - basicsQuery = overrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _start: this.getBinStart(override), - _end: this.getBinEnd(override) - }); - - binsQuery = [ - overrideBinsQueryTpl({ - _bins: override.bins - }) - ].join(',\n'); - } else { - basicsQuery = basicsQueryTpl({ - _query: _query, - _column: _column - }); - - if (this._shouldOverrideBins(override)) { - binsQuery = [ - overrideBinsQueryTpl({ - _bins: override.bins - }) - ].join(',\n'); - } else { - binsQuery = [ - iqrQueryTpl({ - _query: _query, - _column: _column - }), - binsQueryTpl({ - _query: _query, - _minBins: BIN_MIN_NUMBER, - _maxBins: BIN_MAX_NUMBER - }) - ].join(',\n'); - } - } - - var cteSql = [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }) - ]; - - if (this._columnType === 'float') { - cteSql.push( - infinitiesQueryTpl({ - _query: _query, - _column: _column - }), - nansQueryTpl({ - _query: _query, - _column: _column - }) - ); - } - - var histogramSql = [ - "WITH", - cteSql.join(',\n'), - histogramQueryTpl({ - _isFloatColumn: this._columnType === 'float', - _query: _query, - _column: _column - }) - ].join('\n'); - debug(histogramSql); return callback(null, histogramSql); } + buildNumericHistogramQueryTpl (ctx) { + return ` + WITH + ${filteredQueryTpl(ctx)}, + ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._shouldOverride(ctx._override) ? + overrideBinsQueryTpl(ctx) : + this._shouldOverrideBins(ctx._override) ? + overrideBinsQueryTpl(ctx) : + `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` + }, + ${nullsQueryTpl(ctx)} + ${ctx._isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} + ${histogramQueryTpl(ctx)} + `; + } + _shouldOverride (override) { return override && override.hasOwnProperty('start') && From b63a67a5b89b0d4843636be2e5b1526b558fc49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 18:33:51 +0200 Subject: [PATCH 036/177] Avoid nested ternaries --- lib/cartodb/models/dataview/histogram.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 60d6cd36..3ca87bb7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -112,7 +112,7 @@ const binsQueryTpl = ctx => ` const overrideBinsQueryTpl = ctx => ` __cdb_bins AS ( - SELECT ${ctx._bins} AS __cdb_bins_number + SELECT ${ctx._override.bins} AS __cdb_bins_number ) `; @@ -380,7 +380,6 @@ module.exports = class Histogram extends BaseDataview { _end: this.getBinEnd(override), _minBins: BIN_MIN_NUMBER, _maxBins: BIN_MAX_NUMBER, - _bins: override.bins }); debug(histogramSql); @@ -393,11 +392,9 @@ module.exports = class Histogram extends BaseDataview { WITH ${filteredQueryTpl(ctx)}, ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._shouldOverride(ctx._override) ? + ${this._shouldOverride(ctx._override) || this._shouldOverrideBins(ctx._override) ? overrideBinsQueryTpl(ctx) : - this._shouldOverrideBins(ctx._override) ? - overrideBinsQueryTpl(ctx) : - `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` + `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` }, ${nullsQueryTpl(ctx)} ${ctx._isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} From 63a7ee08d0cf686da39740710e25d754953f4a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 6 Sep 2017 19:15:39 +0200 Subject: [PATCH 037/177] Avoid nested ternaries for date histograms --- lib/cartodb/models/dataview/histogram.js | 64 +++++++----------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 3ca87bb7..c4b53423 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -413,9 +413,19 @@ module.exports = class Histogram extends BaseDataview { return override && override.hasOwnProperty('bins'); } + buildDateHistogramQueryTpl (ctx) { + return ` + WITH + ${ctx._override && ctx._override.hasOwnProperty('start') && ctx._override.hasOwnProperty('end') ? + dateOverrideBasicsQueryTpl(ctx) : + dateBasicsQueryTpl(ctx)}, + ${dateBinsQueryTpl(ctx)}, + ${nullsQueryTpl(ctx)} + ${dateHistogramQueryTpl(ctx)} + `; + } + _buildDateHistogramQuery (psql, override, callback) { - var _column = this.column; - var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; @@ -438,54 +448,16 @@ module.exports = class Histogram extends BaseDataview { return null; } - var dateBasicsQuery; - - if (override && override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - dateBasicsQuery = dateOverrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _start: this.getBinStart(override), - _end: this.getBinEnd(override), - _offset: this.parseOffset(_offset, _aggregation) - }); - } else { - dateBasicsQuery = dateBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _offset: this.parseOffset(_offset, _aggregation) - }); - } - - var dateBinsQuery = [ - dateBinsQueryTpl({ - _aggregation: _aggregation - }) - ].join(',\n'); - - var nullsQuery = nullsQueryTpl({ - _query: _query, - _column: _column - }); - - var dateHistogramQuery = dateHistogramQueryTpl({ - _query: _query, - _column: _column, + const histogramSql = this.buildDateHistogramQueryTpl({ + _override: override, + _query: this.query, + _column: this.column, _aggregation: _aggregation, + _start: this.getBinStart(override), + _end: this.getBinEnd(override), _offset: this.parseOffset(_offset, _aggregation) }); - var histogramSql = [ - "WITH", - [ - dateBasicsQuery, - dateBinsQuery, - nullsQuery - ].join(',\n'), - dateHistogramQuery - ].join('\n'); - debug(histogramSql); return callback(null, histogramSql); From 7e099be13474cd1050fba7367c07209829abd1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 8 Sep 2017 10:29:54 +0200 Subject: [PATCH 038/177] Add specific implementations of histograms based on column type; still not used --- lib/cartodb/models/dataview/date-histogram.js | 441 ++++++++++++++++++ .../models/dataview/numeric-histogram.js | 369 +++++++++++++++ 2 files changed, 810 insertions(+) create mode 100644 lib/cartodb/models/dataview/date-histogram.js create mode 100644 lib/cartodb/models/dataview/numeric-histogram.js diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js new file mode 100644 index 00000000..2e55ea2d --- /dev/null +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -0,0 +1,441 @@ +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:histogram'); + +const dateIntervalQueryTpl = ctx => ` + WITH + __cdb_dates AS ( + SELECT + MAX(${ctx.column}::timestamp) AS __cdb_end, + MIN(${ctx.column}::timestamp) AS __cdb_start + FROM (${ctx.query}) __cdb_source + ), + __cdb_interval_in_days AS ( + SELECT + DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days + FROM __cdb_dates + ), + __cdb_interval_in_hours AS ( + SELECT + __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours + FROM __cdb_interval_in_days, __cdb_dates + ), + __cdb_interval_in_minutes AS ( + SELECT + __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes + FROM __cdb_interval_in_hours, __cdb_dates + ), + __cdb_interval_in_seconds AS ( + SELECT + __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds + FROM __cdb_interval_in_minutes, __cdb_dates + ) + SELECT + ROUND(__cdb_days / 365) AS year, + ROUND(__cdb_days / 90) AS quarter, + ROUND(__cdb_days / 30) AS month, + ROUND(__cdb_days / 7) AS week, + __cdb_days AS day, + __cdb_hours AS hour, + __cdb_minutes AS minute, + __cdb_seconds AS second + FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds +`; + +const nullsQueryTpl = ctx => ` + __cdb_nulls AS ( + SELECT + count(*) AS __cdb_nulls_count + FROM (${ctx._query}) __cdb_histogram_nulls + WHERE ${ctx._column} IS NULL + ) +`; + +const dateBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(date_part('epoch', ${ctx._column})) AS __cdb_max_val, + min(date_part('epoch', ${ctx._column})) AS __cdb_min_val, + avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' + ) + ) AS __cdb_start_date, + max(${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}') AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx._query}) __cdb_basics_query + ) +`; + +const dateOverrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx._end}) AS __cdb_max_val, + min(${ctx._start}) AS __cdb_min_val, + avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx._aggregation}', + TO_TIMESTAMP(${ctx._start})::timestamp AT TIME ZONE '${ctx._offset}' + ) + ) AS __cdb_start_date, + max( + TO_TIMESTAMP(${ctx._end})::timestamp AT TIME ZONE '${ctx._offset}' + ) AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx._query}) __cdb_basics_query + ) +`; + +const dateBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + __cdb_bins_array, + ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number + FROM ( + SELECT + ARRAY( + SELECT GENERATE_SERIES( + __cdb_start_date::timestamptz, + __cdb_end_date::timestamptz, + ${ctx._aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx._aggregation}'::interval`} + ) + ) AS __cdb_bins_array + FROM __cdb_basics + ) __cdb_bins_array_query + ) +`; + +const dateHistogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET( + ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}', + __cdb_bins_array + ), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min( + date_part( + 'epoch', + date_trunc( + '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' + ) AT TIME ZONE '${ctx._offset}' + ) + )::numeric AS timestamp, + date_part('epoch', __cdb_start_date)::numeric AS timestamp_start, + min(date_part('epoch', ${ctx._column}))::numeric AS min, + max(date_part('epoch', ${ctx._column}))::numeric AS max, + avg(date_part('epoch', ${ctx._column}))::numeric AS avg, + count(*) AS freq + FROM (${ctx._query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls + WHERE date_part('epoch', ${ctx._column}) IS NOT NULL + GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start + ORDER BY bin +`; + +const MAX_INTERVAL_VALUE = 366; + +const TYPE = 'histogram'; + +const DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + +/** + date_histogram: { + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // auto by default + offset: -7200 // OPTIONAL (UTC offset in seconds) + } + } +*/ +module.exports = class Histogram extends BaseDataview { + constructor (query, options, queries) { + super(); + + if (typeof options.column !== 'string') { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.bins = options.bins; + this.aggregation = options.aggregation; + this.offset = options.offset; + + this._columnType = null; + } + + sql (psql, override, callback) { + var self = this; + + if (!callback) { + callback = override; + override = {}; + } + + if (this._columnType === null) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + // assume numeric, will fail later + self._columnType = 'numeric'; + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); + } + self.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + if (this.isDateHistogram(override)) { + return this._buildQuery(psql, override, callback); + } + + } + + isDateHistogram (override) { + return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); + } + + buildDateHistogramQueryTpl (ctx) { + return ` + WITH + ${ctx._override && ctx._override.hasOwnProperty('start') && ctx._override.hasOwnProperty('end') ? + dateOverrideBasicsQueryTpl(ctx) : + dateBasicsQueryTpl(ctx)}, + ${dateBinsQueryTpl(ctx)}, + ${nullsQueryTpl(ctx)} + ${dateHistogramQueryTpl(ctx)} + `; + } + + _buildQuery (psql, override, callback) { + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + + if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + + if (_aggregation === 'auto') { + this.getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + override.aggregation = this.aggregation; + this._buildDateHistogramQuery(psql, override, callback); + }.bind(this)); + return null; + } + + const histogramSql = this.buildDateHistogramQueryTpl({ + _override: override, + _query: this.query, + _column: this.column, + _aggregation: _aggregation, + _start: this.getBinStart(override), + _end: this.getBinEnd(override), + _offset: this.parseOffset(_offset, _aggregation) + }); + + debug(histogramSql); + + return callback(null, histogramSql); + } + + getAutomaticAggregation (psql, callback) { + var dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + debug(dateIntervalQuery); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + var aggegations = result.rows[0]; + var aggregation = Object.keys(aggegations) + .map(function (key) { + return { + name: key, + value: aggegations[key] + }; + }) + .reduce(function (closer, current) { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + var closerDiff = MAX_INTERVAL_VALUE - closer.value; + var currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); + } + + format (result, override) { + override = override || {}; + var buckets = []; + + var binsCount = this.getBinsCount(override); + var width = this.getWidth(override); + var binsStart = this.getBinStart(override); + var nulls = 0; + var infinities = 0; + var nans = 0; + var avg; + var timestampStart; + var aggregation; + var offset; + + if (result.rows.length) { + var firstRow = result.rows[0]; + binsCount = firstRow.bins_number; + width = firstRow.bin_width || width; + avg = firstRow.avg_val; + nulls = firstRow.nulls_count; + timestampStart = firstRow.timestamp_start; + infinities = firstRow.infinities_count; + nans = firstRow.nans_count; + binsStart = this.populateBinStart(override, firstRow); + + if (Number.isFinite(timestampStart)) { + aggregation = this.getAggregation(override); + offset = this.getOffset(override); + } + + buckets = result.rows.map(row => ({ bin, min, max, avg, freq, timestamp } = row)); + } + + return { + aggregation: aggregation, + offset: offset, + timestamp_start: timestampStart, + bin_width: width, + bins_count: binsCount, + bins_start: binsStart, + nulls: nulls, + infinities: infinities, + nans: nans, + avg: avg, + bins: buckets + }; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); + } + + getAggregation (override) { + return override && override.aggregation ? override.aggregation : this.aggregation; + } + + getOffset (override) { + if (override && override.offset) { + return override.offset; + } + if (this.offset) { + return this.offset; + } + + return 0; + } + + getBinStart (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.min(override.start, override.end); + } + return override.start || 0; + } + + populateBinStart (override, firstRow) { + var binStart; + + if (firstRow.hasOwnProperty('timestamp')) { + binStart = firstRow.timestamp; + } else if (override.hasOwnProperty('start')) { + binStart = this.getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; + } + + getBinEnd (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.max(override.start, override.end); + } + return override.end || 0; + } + + getBinsCount (override) { + return override.bins || 0; + } + + getWidth (override) { + var width = 0; + var binsCount = override.bins; + + if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { + width = (override.end - override.start) / binsCount; + } + + return width; + } + + parseOffset (offset, aggregation) { + if (!offset) { + return '0'; + } + if (aggregation === 'hour' || aggregation === 'minute') { + return '0'; + } + + var offsetInHours = Math.ceil(offset / 3600); + return '' + offsetInHours; + } +}; diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js new file mode 100644 index 00000000..e5149793 --- /dev/null +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -0,0 +1,369 @@ +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:histogram'); + +const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; + +const filterOutSpecialNumericValues = ctx => ` + ${ctx._column} != 'infinity'::float + AND + ${ctx._column} != '-infinity'::float + AND + ${ctx._column} != 'NaN'::float +`; + +const filteredQueryTpl = ctx => ` + __cdb_filtered_source AS ( + SELECT * + FROM (${ctx._query}) __cdb_filtered_source_query + WHERE ${ctx._column} IS NOT NULL + ${ctx._isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} + ) +`; + +const basicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx._column}) AS __cdb_max_val, min(${ctx._column}) AS __cdb_min_val, + avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; + +const overrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx._end}) AS __cdb_max_val, min(${ctx._start}) AS __cdb_min_val, + avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; + +const iqrQueryTpl = ctx => ` + __cdb_iqrange AS ( + SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr + FROM ( + SELECT quartile, max(_cdb_iqr_column) AS quartile_max from ( + SELECT ${ctx._column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx._column} + ) AS quartile + FROM __cdb_filtered_source) _cdb_quartiles + WHERE quartile = 1 or quartile = 3 + GROUP BY quartile + ) __cdb_iqr + ) +`; + +const binsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 + THEN 1 + ELSE GREATEST( + LEAST(${ctx._minBins}, CAST(__cdb_total_rows AS INT)), + LEAST( + CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT), + ${ctx._maxBins} + ) + ) + END AS __cdb_bins_number + FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source + LIMIT 1 + ) +`; + +const overrideBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT ${ctx._override.bins} AS __cdb_bins_number + ) +`; + +const nullsQueryTpl = ctx => ` + __cdb_nulls AS ( + SELECT + count(*) AS __cdb_nulls_count + FROM (${ctx._query}) __cdb_histogram_nulls + WHERE ${ctx._column} IS NULL + ) +`; + +const infinitiesQueryTpl = ctx => ` + __cdb_infinities AS ( + SELECT + count(*) AS __cdb_infinities_count + FROM (${ctx._query}) __cdb_infinities_query + WHERE + ${ctx._column} = 'infinity'::float + OR + ${ctx._column} = '-infinity'::float + ) +`; + +const nansQueryTpl = ctx => ` + __cdb_nans AS ( + SELECT + count(*) AS __cdb_nans_count + FROM (${ctx._query}) __cdb_nans_query + WHERE ${ctx._column} = 'NaN'::float + ) +`; + +const specialNumericValuesColumnDefinitionTpl = () => ` + __cdb_infinities_count AS infinities_count, + __cdb_nans_count AS nans_count +`; + +const specialNumericValuesCTETpl = () => ` + __cdb_infinities, __cdb_nans +`; + +const specialNumericValuesColumnTpl = () => ` + infinities_count, nans_count +`; + +const histogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + ${ctx._isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} + __cdb_avg_val AS avg_val, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET(${ctx._column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min(${ctx._column})::numeric AS min, + max(${ctx._column})::numeric AS max, + avg(${ctx._column})::numeric AS avg, + count(*) AS freq + FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins + ${ctx._isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} + GROUP BY bin, bins_number, bin_width, nulls_count, avg_val + ${ctx._isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} + ORDER BY bin +`; + +const BIN_MIN_NUMBER = 6; +const BIN_MAX_NUMBER = 48; + +const TYPE = 'histogram'; + +/** +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} +*/ +module.exports = class NumericHistogram extends BaseDataview { + constructor (query, options, queries) { + super(); + + if (typeof options.column !== 'string') { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.bins = options.bins; + this.aggregation = options.aggregation; + this.offset = options.offset; + + this._columnType = null; + } + + sql (psql, override, callback) { + var self = this; + + if (!callback) { + callback = override; + override = {}; + } + + if (this._columnType === null) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + // assume numeric, will fail later + self._columnType = 'numeric'; + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); + } + self.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + this._buildQuery(psql, override, callback); + } + + + _buildQuery (psql, override, callback) { + const histogramSql = this.buildNumericHistogramQueryTpl({ + _override: override, + _column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, + _isFloatColumn: this._columnType === 'float', + _query: this.query, + _start: this.getBinStart(override), + _end: this.getBinEnd(override), + _minBins: BIN_MIN_NUMBER, + _maxBins: BIN_MAX_NUMBER, + }); + + debug(histogramSql); + + return callback(null, histogramSql); + } + + buildNumericHistogramQueryTpl (ctx) { + return ` + WITH + ${filteredQueryTpl(ctx)}, + ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._shouldOverrideRange(ctx._override) || this._shouldOverrideBins(ctx._override) ? + overrideBinsQueryTpl(ctx) : + `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` + }, + ${nullsQueryTpl(ctx)} + ${ctx._isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} + ${histogramQueryTpl(ctx)} + `; + } + + _shouldOverride (override) { + return this._shouldOverrideRange(override) && this._shouldOverrideBins(override); + } + + _shouldOverrideRange (override) { + return override && override.hasOwnProperty('start') && override.hasOwnProperty('end'); + } + + _shouldOverrideBins (override) { + return override && override.hasOwnProperty('bins'); + } + + format (result, override) { + override = override || {}; + var buckets = []; + + var binsCount = this.getBinsCount(override); + var width = this.getWidth(override); + var binsStart = this.getBinStart(override); + var nulls = 0; + var infinities = 0; + var nans = 0; + var avg; + var timestampStart; + var aggregation; + var offset; + + if (result.rows.length) { + var firstRow = result.rows[0]; + binsCount = firstRow.bins_number; + width = firstRow.bin_width || width; + avg = firstRow.avg_val; + nulls = firstRow.nulls_count; + timestampStart = firstRow.timestamp_start; + infinities = firstRow.infinities_count; + nans = firstRow.nans_count; + binsStart = this.populateBinStart(override, firstRow); + + if (Number.isFinite(timestampStart)) { + aggregation = this.getAggregation(override); + offset = this.getOffset(override); + } + + buckets = result.rows.map(row => ({ bin, min, max, avg, freq } = row)); + } + + return { + aggregation: aggregation, + offset: offset, + timestamp_start: timestampStart, + bin_width: width, + bins_count: binsCount, + bins_start: binsStart, + nulls: nulls, + infinities: infinities, + nans: nans, + avg: avg, + bins: buckets + }; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); + } + + getAggregation (override) { + return override && override.aggregation ? override.aggregation : this.aggregation; + } + + getOffset (override) { + if (override && override.offset) { + return override.offset; + } + if (this.offset) { + return this.offset; + } + + return 0; + } + + getBinStart (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.min(override.start, override.end); + } + return override.start || 0; + } + + populateBinStart (override, firstRow) { + var binStart; + + if (override.hasOwnProperty('start')) { + binStart = this.getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; + } + + getBinEnd (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.max(override.start, override.end); + } + return override.end || 0; + } + + getBinsCount (override) { + return override.bins || 0; + } + + getWidth (override) { + var width = 0; + var binsCount = override.bins; + + if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { + width = (override.end - override.start) / binsCount; + } + + return width; + } +}; From 8ce25d958c1e9b1a9ddac8110d13841ea2dad036 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 11:53:47 +0200 Subject: [PATCH 039/177] fix namespace for debugging --- lib/cartodb/models/dataview/date-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index 2e55ea2d..eafd5f34 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -1,5 +1,5 @@ const BaseDataview = require('./base'); -const debug = require('debug')('windshaft:dataview:histogram'); +const debug = require('debug')('windshaft:dataview:date-histogram'); const dateIntervalQueryTpl = ctx => ` WITH From 157946cc42ecb3c28b51646f7860daf3b194d3b7 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 11:54:30 +0200 Subject: [PATCH 040/177] Rename DateHistogram class --- lib/cartodb/models/dataview/date-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index eafd5f34..c8f59b1b 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -168,7 +168,7 @@ const DATE_AGGREGATIONS = { } } */ -module.exports = class Histogram extends BaseDataview { +module.exports = class DateHistogram extends BaseDataview { constructor (query, options, queries) { super(); From a3112aa929d79fcf1980495005b08b7aac8e2a0e Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 11:55:28 +0200 Subject: [PATCH 041/177] fix function name in recursion --- lib/cartodb/models/dataview/date-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index c8f59b1b..f7d11c67 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -248,7 +248,7 @@ module.exports = class DateHistogram extends BaseDataview { this.aggregation = aggregation; } override.aggregation = this.aggregation; - this._buildDateHistogramQuery(psql, override, callback); + this._buildQuery(psql, override, callback); }.bind(this)); return null; } From 347dea8f6656f2429a9c4f456f0c44b30fedc57f Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:01:15 +0200 Subject: [PATCH 042/177] naming private functions --- lib/cartodb/models/dataview/date-histogram.js | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index f7d11c67..efd3f4be 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -208,17 +208,17 @@ module.exports = class DateHistogram extends BaseDataview { return null; } - if (this.isDateHistogram(override)) { + if (this._isDateHistogram(override)) { return this._buildQuery(psql, override, callback); } } - isDateHistogram (override) { + _isDateHistogram (override) { return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); } - buildDateHistogramQueryTpl (ctx) { + _buildDateHistogramQueryTpl (ctx) { return ` WITH ${ctx._override && ctx._override.hasOwnProperty('start') && ctx._override.hasOwnProperty('end') ? @@ -241,7 +241,7 @@ module.exports = class DateHistogram extends BaseDataview { } if (_aggregation === 'auto') { - this.getAutomaticAggregation(psql, function (err, aggregation) { + this._getAutomaticAggregation(psql, function (err, aggregation) { if (err || aggregation === 'none') { this.aggregation = 'day'; } else { @@ -253,14 +253,14 @@ module.exports = class DateHistogram extends BaseDataview { return null; } - const histogramSql = this.buildDateHistogramQueryTpl({ + const histogramSql = this._buildDateHistogramQueryTpl({ _override: override, _query: this.query, _column: this.column, _aggregation: _aggregation, - _start: this.getBinStart(override), - _end: this.getBinEnd(override), - _offset: this.parseOffset(_offset, _aggregation) + _start: this._getBinStart(override), + _end: this._getBinEnd(override), + _offset: this._parseOffset(_offset, _aggregation) }); debug(histogramSql); @@ -268,7 +268,7 @@ module.exports = class DateHistogram extends BaseDataview { return callback(null, histogramSql); } - getAutomaticAggregation (psql, callback) { + _getAutomaticAggregation (psql, callback) { var dateIntervalQuery = dateIntervalQueryTpl({ query: this.query, column: this.column @@ -312,9 +312,9 @@ module.exports = class DateHistogram extends BaseDataview { override = override || {}; var buckets = []; - var binsCount = this.getBinsCount(override); - var width = this.getWidth(override); - var binsStart = this.getBinStart(override); + var binsCount = this._getBinsCount(override); + var width = this._getWidth(override); + var binsStart = this._getBinStart(override); var nulls = 0; var infinities = 0; var nans = 0; @@ -332,11 +332,11 @@ module.exports = class DateHistogram extends BaseDataview { timestampStart = firstRow.timestamp_start; infinities = firstRow.infinities_count; nans = firstRow.nans_count; - binsStart = this.populateBinStart(override, firstRow); + binsStart = this._populateBinStart(override, firstRow); if (Number.isFinite(timestampStart)) { - aggregation = this.getAggregation(override); - offset = this.getOffset(override); + aggregation = this._getAggregation(override); + offset = this._getOffset(override); } buckets = result.rows.map(row => ({ bin, min, max, avg, freq, timestamp } = row)); @@ -357,11 +357,11 @@ module.exports = class DateHistogram extends BaseDataview { }; } - getType () { + _getType () { return TYPE; } - toString () { + _toString () { return JSON.stringify({ _type: TYPE, _column: this.column, @@ -369,11 +369,11 @@ module.exports = class DateHistogram extends BaseDataview { }); } - getAggregation (override) { + _getAggregation (override) { return override && override.aggregation ? override.aggregation : this.aggregation; } - getOffset (override) { + _getOffset (override) { if (override && override.offset) { return override.offset; } @@ -384,20 +384,20 @@ module.exports = class DateHistogram extends BaseDataview { return 0; } - getBinStart (override) { + _getBinStart (override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); } return override.start || 0; } - populateBinStart (override, firstRow) { + _populateBinStart (override, firstRow) { var binStart; if (firstRow.hasOwnProperty('timestamp')) { binStart = firstRow.timestamp; } else if (override.hasOwnProperty('start')) { - binStart = this.getBinStart(override); + binStart = this._getBinStart(override); } else { binStart = firstRow.min; } @@ -405,18 +405,18 @@ module.exports = class DateHistogram extends BaseDataview { return binStart; } - getBinEnd (override) { + _getBinEnd (override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.max(override.start, override.end); } return override.end || 0; } - getBinsCount (override) { + _getBinsCount (override) { return override.bins || 0; } - getWidth (override) { + _getWidth (override) { var width = 0; var binsCount = override.bins; @@ -427,7 +427,7 @@ module.exports = class DateHistogram extends BaseDataview { return width; } - parseOffset (offset, aggregation) { + _parseOffset (offset, aggregation) { if (!offset) { return '0'; } From 3441ad6aa96568b6c14ecfb8d1407275f98c2bd5 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:04:25 +0200 Subject: [PATCH 043/177] rename _buildDateHistogramQueryTpl by _buildQueryTpl --- lib/cartodb/models/dataview/date-histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index efd3f4be..7dd25683 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -218,7 +218,7 @@ module.exports = class DateHistogram extends BaseDataview { return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); } - _buildDateHistogramQueryTpl (ctx) { + _buildQueryTpl (ctx) { return ` WITH ${ctx._override && ctx._override.hasOwnProperty('start') && ctx._override.hasOwnProperty('end') ? @@ -253,7 +253,7 @@ module.exports = class DateHistogram extends BaseDataview { return null; } - const histogramSql = this._buildDateHistogramQueryTpl({ + const histogramSql = this._buildQueryTpl({ _override: override, _query: this.query, _column: this.column, From db969a51ad7c46061beacc9d4194a9ce28c7bd67 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:07:07 +0200 Subject: [PATCH 044/177] make public some private functions --- lib/cartodb/models/dataview/date-histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index 7dd25683..63b4115e 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -357,11 +357,11 @@ module.exports = class DateHistogram extends BaseDataview { }; } - _getType () { + getType () { return TYPE; } - _toString () { + toString () { return JSON.stringify({ _type: TYPE, _column: this.column, From 7f14785091cf53506176744b615d495646b640c0 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:07:49 +0200 Subject: [PATCH 045/177] fix namespace for debugging --- lib/cartodb/models/dataview/numeric-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index e5149793..cb4d8a58 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -1,5 +1,5 @@ const BaseDataview = require('./base'); -const debug = require('debug')('windshaft:dataview:histogram'); +const debug = require('debug')('windshaft:dataview:numeric-histogram'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; From 38f0e23efe05d1834e2c35eb2d28e981f9b388d2 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:10:31 +0200 Subject: [PATCH 046/177] rename buildNumericHistogramQueryTpl by buildQueryTpl --- lib/cartodb/models/dataview/numeric-histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index cb4d8a58..55368f6d 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -207,7 +207,7 @@ module.exports = class NumericHistogram extends BaseDataview { _buildQuery (psql, override, callback) { - const histogramSql = this.buildNumericHistogramQueryTpl({ + const histogramSql = this.buildQueryTpl({ _override: override, _column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, _isFloatColumn: this._columnType === 'float', @@ -223,7 +223,7 @@ module.exports = class NumericHistogram extends BaseDataview { return callback(null, histogramSql); } - buildNumericHistogramQueryTpl (ctx) { + buildQueryTpl (ctx) { return ` WITH ${filteredQueryTpl(ctx)}, From 4a00a2d6739270aafba54ae44448324a6b94406c Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:15:21 +0200 Subject: [PATCH 047/177] rename buildQueryTpl by _buildQueryTpl --- lib/cartodb/models/dataview/numeric-histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 55368f6d..59400a00 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -207,7 +207,7 @@ module.exports = class NumericHistogram extends BaseDataview { _buildQuery (psql, override, callback) { - const histogramSql = this.buildQueryTpl({ + const histogramSql = this._buildQueryTpl({ _override: override, _column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, _isFloatColumn: this._columnType === 'float', @@ -223,7 +223,7 @@ module.exports = class NumericHistogram extends BaseDataview { return callback(null, histogramSql); } - buildQueryTpl (ctx) { + _buildQueryTpl (ctx) { return ` WITH ${filteredQueryTpl(ctx)}, From a4ebce52db238b850cea0d5b582d75c60aa9a05e Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:16:56 +0200 Subject: [PATCH 048/177] remove unnecessary properties in format function --- lib/cartodb/models/dataview/numeric-histogram.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 59400a00..7beb8976 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -261,9 +261,6 @@ module.exports = class NumericHistogram extends BaseDataview { var infinities = 0; var nans = 0; var avg; - var timestampStart; - var aggregation; - var offset; if (result.rows.length) { var firstRow = result.rows[0]; @@ -271,23 +268,14 @@ module.exports = class NumericHistogram extends BaseDataview { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; - timestampStart = firstRow.timestamp_start; infinities = firstRow.infinities_count; nans = firstRow.nans_count; binsStart = this.populateBinStart(override, firstRow); - if (Number.isFinite(timestampStart)) { - aggregation = this.getAggregation(override); - offset = this.getOffset(override); - } - buckets = result.rows.map(row => ({ bin, min, max, avg, freq } = row)); } return { - aggregation: aggregation, - offset: offset, - timestamp_start: timestampStart, bin_width: width, bins_count: binsCount, bins_start: binsStart, From c863cdd9f69f3194dc1bd3cd0450b518e29a7727 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:17:29 +0200 Subject: [PATCH 049/177] remove getAggregation function --- lib/cartodb/models/dataview/numeric-histogram.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 7beb8976..9fd5bdfd 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -299,10 +299,6 @@ module.exports = class NumericHistogram extends BaseDataview { }); } - getAggregation (override) { - return override && override.aggregation ? override.aggregation : this.aggregation; - } - getOffset (override) { if (override && override.offset) { return override.offset; From fb753e50a25955adc5a97ef30c3ed8270caef1f7 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:19:05 +0200 Subject: [PATCH 050/177] remove getOffset function --- lib/cartodb/models/dataview/numeric-histogram.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 9fd5bdfd..3f1f9840 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -299,17 +299,6 @@ module.exports = class NumericHistogram extends BaseDataview { }); } - getOffset (override) { - if (override && override.offset) { - return override.offset; - } - if (this.offset) { - return this.offset; - } - - return 0; - } - getBinStart (override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); From 78b95d05d000446ed33d35f32c0282a278fac963 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:21:22 +0200 Subject: [PATCH 051/177] make private functions --- .../models/dataview/numeric-histogram.js | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 3f1f9840..56167492 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -212,8 +212,8 @@ module.exports = class NumericHistogram extends BaseDataview { _column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, _isFloatColumn: this._columnType === 'float', _query: this.query, - _start: this.getBinStart(override), - _end: this.getBinEnd(override), + _start: this._getBinStart(override), + _end: this._getBinEnd(override), _minBins: BIN_MIN_NUMBER, _maxBins: BIN_MAX_NUMBER, }); @@ -254,9 +254,9 @@ module.exports = class NumericHistogram extends BaseDataview { override = override || {}; var buckets = []; - var binsCount = this.getBinsCount(override); - var width = this.getWidth(override); - var binsStart = this.getBinStart(override); + var binsCount = this._getBinsCount(override); + var width = this._getWidth(override); + var binsStart = this._getBinStart(override); var nulls = 0; var infinities = 0; var nans = 0; @@ -270,7 +270,7 @@ module.exports = class NumericHistogram extends BaseDataview { nulls = firstRow.nulls_count; infinities = firstRow.infinities_count; nans = firstRow.nans_count; - binsStart = this.populateBinStart(override, firstRow); + binsStart = this._populateBinStart(override, firstRow); buckets = result.rows.map(row => ({ bin, min, max, avg, freq } = row)); } @@ -299,18 +299,18 @@ module.exports = class NumericHistogram extends BaseDataview { }); } - getBinStart (override) { + _getBinStart (override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); } return override.start || 0; } - populateBinStart (override, firstRow) { + _populateBinStart (override, firstRow) { var binStart; if (override.hasOwnProperty('start')) { - binStart = this.getBinStart(override); + binStart = this._getBinStart(override); } else { binStart = firstRow.min; } @@ -318,18 +318,18 @@ module.exports = class NumericHistogram extends BaseDataview { return binStart; } - getBinEnd (override) { + _getBinEnd (override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.max(override.start, override.end); } return override.end || 0; } - getBinsCount (override) { + _getBinsCount (override) { return override.bins || 0; } - getWidth (override) { + _getWidth (override) { var width = 0; var binsCount = override.bins; From 7d42afcdb4c72b34f0bf59d5b4d6f1287fae5d5a Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:22:24 +0200 Subject: [PATCH 052/177] remove unnecessary properties of NumericHistogram --- lib/cartodb/models/dataview/numeric-histogram.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 56167492..ea0beb9a 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -174,9 +174,7 @@ module.exports = class NumericHistogram extends BaseDataview { this.queries = queries; this.column = options.column; this.bins = options.bins; - this.aggregation = options.aggregation; - this.offset = options.offset; - + this._columnType = null; } From 23dd143fa50ebc2ceec88b55d29961f435c5f25e Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 12:48:08 +0200 Subject: [PATCH 053/177] Make Histogram class as context of state pattern --- lib/cartodb/models/dataview/histogram.js | 663 +---------------------- 1 file changed, 24 insertions(+), 639 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index c4b53423..a8cd12ed 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,652 +1,37 @@ -const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:histogram'); - -const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; - -const dateIntervalQueryTpl = ctx => ` - WITH - __cdb_dates AS ( - SELECT - MAX(${ctx.column}::timestamp) AS __cdb_end, - MIN(${ctx.column}::timestamp) AS __cdb_start - FROM (${ctx.query}) __cdb_source - ), - __cdb_interval_in_days AS ( - SELECT - DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days - FROM __cdb_dates - ), - __cdb_interval_in_hours AS ( - SELECT - __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours - FROM __cdb_interval_in_days, __cdb_dates - ), - __cdb_interval_in_minutes AS ( - SELECT - __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes - FROM __cdb_interval_in_hours, __cdb_dates - ), - __cdb_interval_in_seconds AS ( - SELECT - __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds - FROM __cdb_interval_in_minutes, __cdb_dates - ) - SELECT - ROUND(__cdb_days / 365) AS year, - ROUND(__cdb_days / 90) AS quarter, - ROUND(__cdb_days / 30) AS month, - ROUND(__cdb_days / 7) AS week, - __cdb_days AS day, - __cdb_hours AS hour, - __cdb_minutes AS minute, - __cdb_seconds AS second - FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds -`; - -const filterOutSpecialNumericValues = ctx => ` - ${ctx._column} != 'infinity'::float - AND - ${ctx._column} != '-infinity'::float - AND - ${ctx._column} != 'NaN'::float -`; - -const filteredQueryTpl = ctx => ` - __cdb_filtered_source AS ( - SELECT * - FROM (${ctx._query}) __cdb_filtered_source_query - WHERE ${ctx._column} IS NOT NULL - ${ctx._isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} - ) -`; - -const basicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(${ctx._column}) AS __cdb_max_val, min(${ctx._column}) AS __cdb_min_val, - avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows - FROM __cdb_filtered_source - ) -`; - -const overrideBasicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(${ctx._end}) AS __cdb_max_val, min(${ctx._start}) AS __cdb_min_val, - avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows - FROM __cdb_filtered_source - ) -`; - -const iqrQueryTpl = ctx => ` - __cdb_iqrange AS ( - SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr - FROM ( - SELECT quartile, max(_cdb_iqr_column) AS quartile_max from ( - SELECT ${ctx._column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx._column} - ) AS quartile - FROM __cdb_filtered_source) _cdb_quartiles - WHERE quartile = 1 or quartile = 3 - GROUP BY quartile - ) __cdb_iqr - ) -`; - -const binsQueryTpl = ctx => ` - __cdb_bins AS ( - SELECT - CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 - THEN 1 - ELSE GREATEST( - LEAST(${ctx._minBins}, CAST(__cdb_total_rows AS INT)), - LEAST( - CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT), - ${ctx._maxBins} - ) - ) - END AS __cdb_bins_number - FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source - LIMIT 1 - ) -`; - -const overrideBinsQueryTpl = ctx => ` - __cdb_bins AS ( - SELECT ${ctx._override.bins} AS __cdb_bins_number - ) -`; - -const nullsQueryTpl = ctx => ` - __cdb_nulls AS ( - SELECT - count(*) AS __cdb_nulls_count - FROM (${ctx._query}) __cdb_histogram_nulls - WHERE ${ctx._column} IS NULL - ) -`; - -const infinitiesQueryTpl = ctx => ` - __cdb_infinities AS ( - SELECT - count(*) AS __cdb_infinities_count - FROM (${ctx._query}) __cdb_infinities_query - WHERE - ${ctx._column} = 'infinity'::float - OR - ${ctx._column} = '-infinity'::float - ) -`; - -const nansQueryTpl = ctx => ` - __cdb_nans AS ( - SELECT - count(*) AS __cdb_nans_count - FROM (${ctx._query}) __cdb_nans_query - WHERE ${ctx._column} = 'NaN'::float - ) -`; - -const specialNumericValuesColumnDefinitionTpl = () => ` - __cdb_infinities_count AS infinities_count, - __cdb_nans_count AS nans_count -`; - -const specialNumericValuesCTETpl = () => ` - __cdb_infinities, __cdb_nans -`; - -const specialNumericValuesColumnTpl = () => ` - infinities_count, nans_count -`; - -const histogramQueryTpl = ctx => ` - SELECT - (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, - __cdb_bins_number AS bins_number, - __cdb_nulls_count AS nulls_count, - ${ctx._isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} - __cdb_avg_val AS avg_val, - CASE WHEN __cdb_min_val = __cdb_max_val - THEN 0 - ELSE GREATEST( - 1, - LEAST( - WIDTH_BUCKET(${ctx._column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), - __cdb_bins_number - ) - ) - 1 - END AS bin, - min(${ctx._column})::numeric AS min, - max(${ctx._column})::numeric AS max, - avg(${ctx._column})::numeric AS avg, - count(*) AS freq - FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins - ${ctx._isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} - GROUP BY bin, bins_number, bin_width, nulls_count, avg_val - ${ctx._isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} - ORDER BY bin -`; - -const dateBasicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(date_part('epoch', ${ctx._column})) AS __cdb_max_val, - min(date_part('epoch', ${ctx._column})) AS __cdb_min_val, - avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, - min( - date_trunc( - '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' - ) - ) AS __cdb_start_date, - max(${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}') AS __cdb_end_date, - count(1) AS __cdb_total_rows - FROM (${ctx._query}) __cdb_basics_query - ) -`; - -const dateOverrideBasicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(${ctx._end}) AS __cdb_max_val, - min(${ctx._start}) AS __cdb_min_val, - avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, - min( - date_trunc( - '${ctx._aggregation}', - TO_TIMESTAMP(${ctx._start})::timestamp AT TIME ZONE '${ctx._offset}' - ) - ) AS __cdb_start_date, - max( - TO_TIMESTAMP(${ctx._end})::timestamp AT TIME ZONE '${ctx._offset}' - ) AS __cdb_end_date, - count(1) AS __cdb_total_rows - FROM (${ctx._query}) __cdb_basics_query - ) -`; - -const dateBinsQueryTpl = ctx => ` - __cdb_bins AS ( - SELECT - __cdb_bins_array, - ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number - FROM ( - SELECT - ARRAY( - SELECT GENERATE_SERIES( - __cdb_start_date::timestamptz, - __cdb_end_date::timestamptz, - ${ctx._aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx._aggregation}'::interval`} - ) - ) AS __cdb_bins_array - FROM __cdb_basics - ) __cdb_bins_array_query - ) -`; - -const dateHistogramQueryTpl = ctx => ` - SELECT - (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, - __cdb_bins_number AS bins_number, - __cdb_nulls_count AS nulls_count, - CASE WHEN __cdb_min_val = __cdb_max_val - THEN 0 - ELSE GREATEST( - 1, - LEAST( - WIDTH_BUCKET( - ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}', - __cdb_bins_array - ), - __cdb_bins_number - ) - ) - 1 - END AS bin, - min( - date_part( - 'epoch', - date_trunc( - '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' - ) AT TIME ZONE '${ctx._offset}' - ) - )::numeric AS timestamp, - date_part('epoch', __cdb_start_date)::numeric AS timestamp_start, - min(date_part('epoch', ${ctx._column}))::numeric AS min, - max(date_part('epoch', ${ctx._column}))::numeric AS max, - avg(date_part('epoch', ${ctx._column}))::numeric AS avg, - count(*) AS freq - FROM (${ctx._query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls - WHERE date_part('epoch', ${ctx._column}) IS NOT NULL - GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start - ORDER BY bin -`; - -const MAX_INTERVAL_VALUE = 366; -const BIN_MIN_NUMBER = 6; -const BIN_MAX_NUMBER = 48; +const NumericHistogram = require('./numeric-histogram'); +const DateHistogram = require('./date-histogram'); const TYPE = 'histogram'; +const DATE_HISTOGRAM = 'DateHistogram'; +const NUMERIC_HISTOGRAM = 'NumericHistogram'; -const DATE_AGGREGATIONS = { - 'auto': true, - 'minute': true, - 'hour': true, - 'day': true, - 'week': true, - 'month': true, - 'quarter': true, - 'year': true -}; - -/** -Numeric histogram: -{ - type: 'histogram', - options: { - column: 'name', // column data type: numeric - bins: 10 // OPTIONAL - } -} - -Time series: -{ - type: 'histogram', - options: { - column: 'date', // column data type: date - aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) - offset: -7200 // OPTIONAL (UTC offset in seconds) - } - } - */ - -module.exports = class Histogram extends BaseDataview { +module.exports = class Histogram { constructor (query, options, queries) { - super(); - - if (typeof options.column !== 'string') { - throw new Error('Histogram expects `column` in widget options'); + switch (this._getHistogramSubtype(options)) { + case DATE_HISTOGRAM: + debug('Delegating to DateHistogram with options: %j', options) + this.dataview = new DateHistogram(query, options, queries); + break; + case NUMERIC_HISTOGRAM: + debug('Delegating to NumericHistogram with options: %j', options) + this.dataview = new NumericHistogram(query, options, queries); + break; + + default: + throw new Error('Unsupported Histogram type'); } - - this.query = query; - this.queries = queries; - this.column = options.column; - this.bins = options.bins; - this.aggregation = options.aggregation; - this.offset = options.offset; - - this._columnType = null; } - sql (psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; + _getHistogramSubtype (options) { + if(options.bins && !options.aggregation) { + return NUMERIC_HISTOGRAM + } else if(options.aggregation && !options.bins) { + return DATE_HISTOGRAM } - - if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { - // assume numeric, will fail later - self._columnType = 'numeric'; - if (!err && !!type) { - self._columnType = Object.keys(type).find(function (key) { - return type[key]; - }); - } - self.sql(psql, override, callback); - }, true); // use read-only transaction - return null; - } - - this._buildQuery(psql, override, callback); } - isDateHistogram (override) { - return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); - } - - _buildQuery (psql, override, callback) { - - if (this.isDateHistogram(override)) { - return this._buildDateHistogramQuery(psql, override, callback); - } - - const histogramSql = this.buildNumericHistogramQueryTpl({ - _override: override, - _column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, - _isFloatColumn: this._columnType === 'float', - _query: this.query, - _start: this.getBinStart(override), - _end: this.getBinEnd(override), - _minBins: BIN_MIN_NUMBER, - _maxBins: BIN_MAX_NUMBER, - }); - - debug(histogramSql); - - return callback(null, histogramSql); - } - - buildNumericHistogramQueryTpl (ctx) { - return ` - WITH - ${filteredQueryTpl(ctx)}, - ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._shouldOverride(ctx._override) || this._shouldOverrideBins(ctx._override) ? - overrideBinsQueryTpl(ctx) : - `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` - }, - ${nullsQueryTpl(ctx)} - ${ctx._isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} - ${histogramQueryTpl(ctx)} - `; - } - - _shouldOverride (override) { - return override && - override.hasOwnProperty('start') && - override.hasOwnProperty('end') && - override.hasOwnProperty('bins'); - } - - _shouldOverrideBins (override) { - return override && override.hasOwnProperty('bins'); - } - - buildDateHistogramQueryTpl (ctx) { - return ` - WITH - ${ctx._override && ctx._override.hasOwnProperty('start') && ctx._override.hasOwnProperty('end') ? - dateOverrideBasicsQueryTpl(ctx) : - dateBasicsQueryTpl(ctx)}, - ${dateBinsQueryTpl(ctx)}, - ${nullsQueryTpl(ctx)} - ${dateHistogramQueryTpl(ctx)} - `; - } - - _buildDateHistogramQuery (psql, override, callback) { - var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; - - if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { - return callback(new Error('Invalid aggregation value. Valid ones: ' + - Object.keys(DATE_AGGREGATIONS).join(', ') - )); - } - - if (_aggregation === 'auto') { - this.getAutomaticAggregation(psql, function (err, aggregation) { - if (err || aggregation === 'none') { - this.aggregation = 'day'; - } else { - this.aggregation = aggregation; - } - override.aggregation = this.aggregation; - this._buildDateHistogramQuery(psql, override, callback); - }.bind(this)); - return null; - } - - const histogramSql = this.buildDateHistogramQueryTpl({ - _override: override, - _query: this.query, - _column: this.column, - _aggregation: _aggregation, - _start: this.getBinStart(override), - _end: this.getBinEnd(override), - _offset: this.parseOffset(_offset, _aggregation) - }); - - debug(histogramSql); - - return callback(null, histogramSql); - } - - getAutomaticAggregation (psql, callback) { - var dateIntervalQuery = dateIntervalQueryTpl({ - query: this.query, - column: this.column - }); - - debug(dateIntervalQuery); - - psql.query(dateIntervalQuery, function (err, result) { - if (err) { - return callback(err); - } - - var aggegations = result.rows[0]; - var aggregation = Object.keys(aggegations) - .map(function (key) { - return { - name: key, - value: aggegations[key] - }; - }) - .reduce(function (closer, current) { - if (current.value > MAX_INTERVAL_VALUE) { - return closer; - } - - var closerDiff = MAX_INTERVAL_VALUE - closer.value; - var currentDiff = MAX_INTERVAL_VALUE - current.value; - - if (Number.isFinite(current.value) && closerDiff > currentDiff) { - return current; - } - - return closer; - }, { name: 'none', value: -1 }); - - callback(null, aggregation.name); - }); - } - - format (result, override) { - override = override || {}; - var buckets = []; - - var binsCount = this.getBinsCount(override); - var width = this.getWidth(override); - var binsStart = this.getBinStart(override); - var nulls = 0; - var infinities = 0; - var nans = 0; - var avg; - var timestampStart; - var aggregation; - var offset; - - if (result.rows.length) { - var firstRow = result.rows[0]; - binsCount = firstRow.bins_number; - width = firstRow.bin_width || width; - avg = firstRow.avg_val; - nulls = firstRow.nulls_count; - timestampStart = firstRow.timestamp_start; - infinities = firstRow.infinities_count; - nans = firstRow.nans_count; - binsStart = this.populateBinStart(override, firstRow); - - if (Number.isFinite(timestampStart)) { - aggregation = this.getAggregation(override); - offset = this.getOffset(override); - } - - // TODO: after applying strategy we could use specialized .format() - // in order to know what props we want to expose using destructuring assignment - - const blacklisted = [ - 'bins_number', - 'bin_width', - 'nulls_count', - 'infinities_count', - 'nans_count', - 'avg_val', - 'timestamp_start' - ]; - - buckets = result.rows.map(row => Object.keys(row) - .filter((key) => blacklisted.indexOf(key) < 0) - .reduce((newObj, key) => Object.assign(newObj, { [key]: row[key] }), {}) - ); - } - - return { - aggregation: aggregation, - offset: offset, - timestamp_start: timestampStart, - bin_width: width, - bins_count: binsCount, - bins_start: binsStart, - nulls: nulls, - infinities: infinities, - nans: nans, - avg: avg, - bins: buckets - }; - } - - getType () { - return TYPE; - } - - toString () { - return JSON.stringify({ - _type: TYPE, - _column: this.column, - _query: this.query - }); - } - - getAggregation (override) { - return override && override.aggregation ? override.aggregation : this.aggregation; - } - - getOffset (override) { - if (override && override.offset) { - return override.offset; - } - if (this.offset) { - return this.offset; - } - - return 0; - } - - getBinStart (override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.min(override.start, override.end); - } - return override.start || 0; - } - - populateBinStart (override, firstRow) { - var binStart; - - if (firstRow.hasOwnProperty('timestamp')) { - binStart = firstRow.timestamp; - } else if (override.hasOwnProperty('start')) { - binStart = this.getBinStart(override); - } else { - binStart = firstRow.min; - } - - return binStart; - } - - getBinEnd (override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.max(override.start, override.end); - } - return override.end || 0; - } - - getBinsCount (override) { - return override.bins || 0; - } - - getWidth (override) { - var width = 0; - var binsCount = override.bins; - - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; - } - - return width; - } - - parseOffset (offset, aggregation) { - if (!offset) { - return '0'; - } - if (aggregation === 'hour' || aggregation === 'minute') { - return '0'; - } - - var offsetInHours = Math.ceil(offset / 3600); - return '' + offsetInHours; + getResult (psql, override, callback) { + this.dataview.getResult(psql, override, callback); } }; From f75b4312a16ca0c566b4d1585f2a54cd32f07f93 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 15:49:25 +0200 Subject: [PATCH 054/177] Fix undefined while destructuring assignment --- lib/cartodb/models/dataview/numeric-histogram.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index ea0beb9a..5cf6f96e 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -270,7 +270,10 @@ module.exports = class NumericHistogram extends BaseDataview { nans = firstRow.nans_count; binsStart = this._populateBinStart(override, firstRow); - buckets = result.rows.map(row => ({ bin, min, max, avg, freq } = row)); + buckets = result.rows.map(row => { + const { bin, min, max, avg, freq } = row; + return { bin, min, max, avg, freq }; + }); } return { From 1d4935cc9a4992ae2a1a6e79b535915e28ccaa32 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 15:50:01 +0200 Subject: [PATCH 055/177] Fix undefined while destrutcuring assignment --- lib/cartodb/models/dataview/date-histogram.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index 63b4115e..aee5562e 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -339,7 +339,10 @@ module.exports = class DateHistogram extends BaseDataview { offset = this._getOffset(override); } - buckets = result.rows.map(row => ({ bin, min, max, avg, freq, timestamp } = row)); + buckets = result.rows.map(row => { + const { bin, min, max, avg, freq, timestamp } = row; + return { bin, min, max, avg, freq, timestamp }; + }); } return { From 29f32cb9cc7d09e945e7bb15b8026896aab0a38c Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 15:53:00 +0200 Subject: [PATCH 056/177] Expose dataview's methods to bypass concrete overview's implementations --- lib/cartodb/models/dataview/histogram.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index a8cd12ed..2261b73a 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -34,4 +34,24 @@ module.exports = class Histogram { getResult (psql, override, callback) { this.dataview.getResult(psql, override, callback); } + + // In order to keep previous behaviour with overviews, + // we have to expose the following methods to bypass + // the concrete overview implementation + + sql (psql, override, callback) { + this.dataview.sql(psql, override, callback); + } + + format (result, override) { + return this.dataview.format(result, override); + } + + getType () { + return this.dataview.getType(); + } + + toString () { + return this.dataview.toString(); + } }; From 5593d92c4bcedf5fedc11e300cef6ea9c05855d0 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 15:55:23 +0200 Subject: [PATCH 057/177] Do not choose histogram implementation until getResult() --- lib/cartodb/models/dataview/histogram.js | 37 ++++++++++++++---------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 2261b73a..461f0528 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -7,31 +7,38 @@ const DATE_HISTOGRAM = 'DateHistogram'; const NUMERIC_HISTOGRAM = 'NumericHistogram'; module.exports = class Histogram { - constructor (query, options, queries) { - switch (this._getHistogramSubtype(options)) { + constructor (query, options = {}, queries) { + this.query = query; + this.options = options; + this.queries = queries; + + this.dataview = this._getHistogramImplemetation(); + } + + _getHistogramImplemetation (override) { + switch (this._getHistogramSubtype(override)) { case DATE_HISTOGRAM: - debug('Delegating to DateHistogram with options: %j', options) - this.dataview = new DateHistogram(query, options, queries); + debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override) + return new DateHistogram(this.query, this.options, this.queries); break; case NUMERIC_HISTOGRAM: - debug('Delegating to NumericHistogram with options: %j', options) - this.dataview = new NumericHistogram(query, options, queries); + debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override) + return new NumericHistogram(this.query, this.options, this.queries); break; - default: throw new Error('Unsupported Histogram type'); } - } - - _getHistogramSubtype (options) { - if(options.bins && !options.aggregation) { - return NUMERIC_HISTOGRAM - } else if(options.aggregation && !options.bins) { - return DATE_HISTOGRAM + } + + _getHistogramSubtype (override = {}) { + if(this.options.aggregation !== undefined || override.aggregation !== undefined) { + return DATE_HISTOGRAM; } + return NUMERIC_HISTOGRAM; } - + getResult (psql, override, callback) { + this.dataview = this._getHistogramImplemetation(override); this.dataview.getResult(psql, override, callback); } From 5e07cc2ad1ae9fed6a4943a296e3d5685d79c8ff Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 16:06:54 +0200 Subject: [PATCH 058/177] Remove unneeded condittion --- lib/cartodb/models/dataview/date-histogram.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index aee5562e..a38bd792 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -334,10 +334,8 @@ module.exports = class DateHistogram extends BaseDataview { nans = firstRow.nans_count; binsStart = this._populateBinStart(override, firstRow); - if (Number.isFinite(timestampStart)) { - aggregation = this._getAggregation(override); - offset = this._getOffset(override); - } + aggregation = this._getAggregation(override); + offset = this._getOffset(override); buckets = result.rows.map(row => { const { bin, min, max, avg, freq, timestamp } = row; From 92f6f59e079a16d59a08e9e2e87ba5474e396bb0 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 16:09:57 +0200 Subject: [PATCH 059/177] Fix jshint style errors --- lib/cartodb/models/dataview/histogram.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 461f0528..e4179df6 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -2,32 +2,35 @@ const debug = require('debug')('windshaft:dataview:histogram'); const NumericHistogram = require('./numeric-histogram'); const DateHistogram = require('./date-histogram'); -const TYPE = 'histogram'; const DATE_HISTOGRAM = 'DateHistogram'; const NUMERIC_HISTOGRAM = 'NumericHistogram'; module.exports = class Histogram { - constructor (query, options = {}, queries) { + constructor (query, options, queries) { this.query = query; - this.options = options; + this.options = options ||{}; this.queries = queries; this.dataview = this._getHistogramImplemetation(); } _getHistogramImplemetation (override) { + let implementation = null; + switch (this._getHistogramSubtype(override)) { case DATE_HISTOGRAM: - debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override) - return new DateHistogram(this.query, this.options, this.queries); + debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override); + implementation = new DateHistogram(this.query, this.options, this.queries); break; case NUMERIC_HISTOGRAM: - debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override) - return new NumericHistogram(this.query, this.options, this.queries); + debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override); + implementation = new NumericHistogram(this.query, this.options, this.queries); break; default: throw new Error('Unsupported Histogram type'); } + + return implementation; } _getHistogramSubtype (override = {}) { From 25ef2610aa2e0266562621a4b5df122fc01d5cb3 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 17:43:10 +0200 Subject: [PATCH 060/177] Varible declarations to let/const --- lib/cartodb/models/dataview/date-histogram.js | 47 +++++++++---------- .../models/dataview/numeric-histogram.js | 29 +++++------- 2 files changed, 34 insertions(+), 42 deletions(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index a38bd792..0e11befe 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -1,5 +1,4 @@ const BaseDataview = require('./base'); -const debug = require('debug')('windshaft:dataview:date-histogram'); const dateIntervalQueryTpl = ctx => ` WITH @@ -187,7 +186,7 @@ module.exports = class DateHistogram extends BaseDataview { } sql (psql, override, callback) { - var self = this; + let self = this; if (!callback) { callback = override; @@ -231,8 +230,8 @@ module.exports = class DateHistogram extends BaseDataview { } _buildQuery (psql, override, callback) { - var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + const _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + const _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { return callback(new Error('Invalid aggregation value. Valid ones: ' + @@ -263,26 +262,22 @@ module.exports = class DateHistogram extends BaseDataview { _offset: this._parseOffset(_offset, _aggregation) }); - debug(histogramSql); - return callback(null, histogramSql); } _getAutomaticAggregation (psql, callback) { - var dateIntervalQuery = dateIntervalQueryTpl({ + const dateIntervalQuery = dateIntervalQueryTpl({ query: this.query, column: this.column }); - debug(dateIntervalQuery); - psql.query(dateIntervalQuery, function (err, result) { if (err) { return callback(err); } - var aggegations = result.rows[0]; - var aggregation = Object.keys(aggegations) + const aggegations = result.rows[0]; + const aggregation = Object.keys(aggegations) .map(function (key) { return { name: key, @@ -310,21 +305,21 @@ module.exports = class DateHistogram extends BaseDataview { format (result, override) { override = override || {}; - var buckets = []; + let buckets = []; - var binsCount = this._getBinsCount(override); - var width = this._getWidth(override); - var binsStart = this._getBinStart(override); - var nulls = 0; - var infinities = 0; - var nans = 0; - var avg; - var timestampStart; - var aggregation; - var offset; + let binsCount = this._getBinsCount(override); + let width = this._getWidth(override); + let binsStart = this._getBinStart(override); + let nulls = 0; + let infinities = 0; + let nans = 0; + let avg; + let timestampStart; + let aggregation; + let offset; if (result.rows.length) { - var firstRow = result.rows[0]; + const firstRow = result.rows[0]; binsCount = firstRow.bins_number; width = firstRow.bin_width || width; avg = firstRow.avg_val; @@ -418,8 +413,8 @@ module.exports = class DateHistogram extends BaseDataview { } _getWidth (override) { - var width = 0; - var binsCount = override.bins; + let width = 0; + const binsCount = override.bins; if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { width = (override.end - override.start) / binsCount; @@ -436,7 +431,7 @@ module.exports = class DateHistogram extends BaseDataview { return '0'; } - var offsetInHours = Math.ceil(offset / 3600); + const offsetInHours = Math.ceil(offset / 3600); return '' + offsetInHours; } }; diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 5cf6f96e..891de107 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -1,5 +1,4 @@ const BaseDataview = require('./base'); -const debug = require('debug')('windshaft:dataview:numeric-histogram'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; @@ -179,7 +178,7 @@ module.exports = class NumericHistogram extends BaseDataview { } sql (psql, override, callback) { - var self = this; + let self = this; if (!callback) { callback = override; @@ -216,8 +215,6 @@ module.exports = class NumericHistogram extends BaseDataview { _maxBins: BIN_MAX_NUMBER, }); - debug(histogramSql); - return callback(null, histogramSql); } @@ -250,18 +247,18 @@ module.exports = class NumericHistogram extends BaseDataview { format (result, override) { override = override || {}; - var buckets = []; + let buckets = []; - var binsCount = this._getBinsCount(override); - var width = this._getWidth(override); - var binsStart = this._getBinStart(override); - var nulls = 0; - var infinities = 0; - var nans = 0; - var avg; + let binsCount = this._getBinsCount(override); + let width = this._getWidth(override); + let binsStart = this._getBinStart(override); + let nulls = 0; + let infinities = 0; + let nans = 0; + let avg; if (result.rows.length) { - var firstRow = result.rows[0]; + const firstRow = result.rows[0]; binsCount = firstRow.bins_number; width = firstRow.bin_width || width; avg = firstRow.avg_val; @@ -308,7 +305,7 @@ module.exports = class NumericHistogram extends BaseDataview { } _populateBinStart (override, firstRow) { - var binStart; + let binStart; if (override.hasOwnProperty('start')) { binStart = this._getBinStart(override); @@ -331,8 +328,8 @@ module.exports = class NumericHistogram extends BaseDataview { } _getWidth (override) { - var width = 0; - var binsCount = override.bins; + let width = 0; + const binsCount = override.bins; if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { width = (override.end - override.start) / binsCount; From 2650c3b3e6a67aeee7a38b16b161780997136a1e Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 17:51:02 +0200 Subject: [PATCH 061/177] removing self=this assignment --- lib/cartodb/models/dataview/date-histogram.js | 10 ++++------ lib/cartodb/models/dataview/numeric-histogram.js | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index 0e11befe..8315cd81 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -186,23 +186,21 @@ module.exports = class DateHistogram extends BaseDataview { } sql (psql, override, callback) { - let self = this; - if (!callback) { callback = override; override = {}; } if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { // assume numeric, will fail later - self._columnType = 'numeric'; + this._columnType = 'numeric'; if (!err && !!type) { - self._columnType = Object.keys(type).find(function (key) { + this._columnType = Object.keys(type).find(function (key) { return type[key]; }); } - self.sql(psql, override, callback); + this.sql(psql, override, callback); }, true); // use read-only transaction return null; } diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 891de107..4968bd58 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -178,23 +178,21 @@ module.exports = class NumericHistogram extends BaseDataview { } sql (psql, override, callback) { - let self = this; - if (!callback) { callback = override; override = {}; } if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { // assume numeric, will fail later - self._columnType = 'numeric'; + this._columnType = 'numeric'; if (!err && !!type) { - self._columnType = Object.keys(type).find(function (key) { + this._columnType = Object.keys(type).find(function (key) { return type[key]; }); } - self.sql(psql, override, callback); + this.sql(psql, override, callback); }, true); // use read-only transaction return null; } From a20900210d431688a84955e77cf3226de969eb9c Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 17:54:25 +0200 Subject: [PATCH 062/177] removing unneeded _isDateHistogram function --- lib/cartodb/models/dataview/date-histogram.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/date-histogram.js index 8315cd81..a0ba927c 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/date-histogram.js @@ -205,14 +205,7 @@ module.exports = class DateHistogram extends BaseDataview { return null; } - if (this._isDateHistogram(override)) { - return this._buildQuery(psql, override, callback); - } - - } - - _isDateHistogram (override) { - return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); + return this._buildQuery(psql, override, callback); } _buildQueryTpl (ctx) { From e4a20fa9545634da597adfd26edd908ec0c163b4 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 8 Sep 2017 17:57:35 +0200 Subject: [PATCH 063/177] adding forgotten return --- lib/cartodb/models/dataview/numeric-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/numeric-histogram.js index 4968bd58..da6792dc 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/numeric-histogram.js @@ -197,7 +197,7 @@ module.exports = class NumericHistogram extends BaseDataview { return null; } - this._buildQuery(psql, override, callback); + return this._buildQuery(psql, override, callback); } From 5e9b2e45c7a1698e0fa617f2c9d9fb6bf825dbe7 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 11 Sep 2017 11:30:09 +0200 Subject: [PATCH 064/177] creating HistogramBase with the common functions of NumericHistogram and DateHistogram --- lib/cartodb/models/dataview/histogram.js | 4 +- .../{ => histograms}/date-histogram.js | 77 +------------------ .../dataview/histograms/histogramBase.js | 77 +++++++++++++++++++ .../{ => histograms}/numeric-histogram.js | 76 +----------------- 4 files changed, 86 insertions(+), 148 deletions(-) rename lib/cartodb/models/dataview/{ => histograms}/date-histogram.js (85%) create mode 100644 lib/cartodb/models/dataview/histograms/histogramBase.js rename lib/cartodb/models/dataview/{ => histograms}/numeric-histogram.js (79%) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index e4179df6..7e0f71bf 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,6 +1,6 @@ const debug = require('debug')('windshaft:dataview:histogram'); -const NumericHistogram = require('./numeric-histogram'); -const DateHistogram = require('./date-histogram'); +const NumericHistogram = require('./histograms/numeric-histogram'); +const DateHistogram = require('./histograms/date-histogram'); const DATE_HISTOGRAM = 'DateHistogram'; const NUMERIC_HISTOGRAM = 'NumericHistogram'; diff --git a/lib/cartodb/models/dataview/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js similarity index 85% rename from lib/cartodb/models/dataview/date-histogram.js rename to lib/cartodb/models/dataview/histograms/date-histogram.js index a0ba927c..064de0ae 100644 --- a/lib/cartodb/models/dataview/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -1,4 +1,4 @@ -const BaseDataview = require('./base'); +const HistogramBase = require('./histogramBase'); const dateIntervalQueryTpl = ctx => ` WITH @@ -144,8 +144,6 @@ const dateHistogramQueryTpl = ctx => ` const MAX_INTERVAL_VALUE = 366; -const TYPE = 'histogram'; - const DATE_AGGREGATIONS = { 'auto': true, 'minute': true, @@ -167,45 +165,12 @@ const DATE_AGGREGATIONS = { } } */ -module.exports = class DateHistogram extends BaseDataview { +module.exports = class DateHistogram extends HistogramBase { constructor (query, options, queries) { - super(); - - if (typeof options.column !== 'string') { - throw new Error('Histogram expects `column` in widget options'); - } - - this.query = query; - this.queries = queries; - this.column = options.column; - this.bins = options.bins; + super(query, options, queries); + this.aggregation = options.aggregation; this.offset = options.offset; - - this._columnType = null; - } - - sql (psql, override, callback) { - if (!callback) { - callback = override; - override = {}; - } - - if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { - // assume numeric, will fail later - this._columnType = 'numeric'; - if (!err && !!type) { - this._columnType = Object.keys(type).find(function (key) { - return type[key]; - }); - } - this.sql(psql, override, callback); - }, true); // use read-only transaction - return null; - } - - return this._buildQuery(psql, override, callback); } _buildQueryTpl (ctx) { @@ -344,18 +309,6 @@ module.exports = class DateHistogram extends BaseDataview { }; } - getType () { - return TYPE; - } - - toString () { - return JSON.stringify({ - _type: TYPE, - _column: this.column, - _query: this.query - }); - } - _getAggregation (override) { return override && override.aggregation ? override.aggregation : this.aggregation; } @@ -392,28 +345,6 @@ module.exports = class DateHistogram extends BaseDataview { return binStart; } - _getBinEnd (override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.max(override.start, override.end); - } - return override.end || 0; - } - - _getBinsCount (override) { - return override.bins || 0; - } - - _getWidth (override) { - let width = 0; - const binsCount = override.bins; - - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; - } - - return width; - } - _parseOffset (offset, aggregation) { if (!offset) { return '0'; diff --git a/lib/cartodb/models/dataview/histograms/histogramBase.js b/lib/cartodb/models/dataview/histograms/histogramBase.js new file mode 100644 index 00000000..dc6c40d6 --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/histogramBase.js @@ -0,0 +1,77 @@ +const BaseDataview = require('../base'); + +const TYPE = 'histogram'; + +module.exports = class HistogramBase extends BaseDataview { + constructor (query, options, queries) { + super(); + + if (typeof options.column !== 'string') { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; + } + + sql (psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + if (this._columnType === null) { + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { + // assume numeric, will fail later + this._columnType = 'numeric'; + if (!err && !!type) { + this._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); + } + this.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + return this._buildQuery(psql, override, callback); + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); + } + + _getBinEnd (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.max(override.start, override.end); + } + return override.end || 0; + } + + _getBinsCount (override) { + return override.bins || 0; + } + + _getWidth (override) { + let width = 0; + const binsCount = override.bins; + + if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { + width = (override.end - override.start) / binsCount; + } + + return width; + } +}; \ No newline at end of file diff --git a/lib/cartodb/models/dataview/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js similarity index 79% rename from lib/cartodb/models/dataview/numeric-histogram.js rename to lib/cartodb/models/dataview/histograms/numeric-histogram.js index da6792dc..2c6ac8e6 100644 --- a/lib/cartodb/models/dataview/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -1,4 +1,4 @@ -const BaseDataview = require('./base'); +const HistogramBase = require('./histogramBase'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; @@ -149,8 +149,6 @@ const histogramQueryTpl = ctx => ` const BIN_MIN_NUMBER = 6; const BIN_MAX_NUMBER = 48; -const TYPE = 'histogram'; - /** Numeric histogram: { @@ -161,46 +159,11 @@ Numeric histogram: } } */ -module.exports = class NumericHistogram extends BaseDataview { +module.exports = class NumericHistogram extends HistogramBase { constructor (query, options, queries) { - super(); - - if (typeof options.column !== 'string') { - throw new Error('Histogram expects `column` in widget options'); - } - - this.query = query; - this.queries = queries; - this.column = options.column; - this.bins = options.bins; - - this._columnType = null; + super(query, options, queries); } - sql (psql, override, callback) { - if (!callback) { - callback = override; - override = {}; - } - - if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { - // assume numeric, will fail later - this._columnType = 'numeric'; - if (!err && !!type) { - this._columnType = Object.keys(type).find(function (key) { - return type[key]; - }); - } - this.sql(psql, override, callback); - }, true); // use read-only transaction - return null; - } - - return this._buildQuery(psql, override, callback); - } - - _buildQuery (psql, override, callback) { const histogramSql = this._buildQueryTpl({ _override: override, @@ -283,18 +246,6 @@ module.exports = class NumericHistogram extends BaseDataview { }; } - getType () { - return TYPE; - } - - toString () { - return JSON.stringify({ - _type: TYPE, - _column: this.column, - _query: this.query - }); - } - _getBinStart (override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); @@ -314,25 +265,4 @@ module.exports = class NumericHistogram extends BaseDataview { return binStart; } - _getBinEnd (override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.max(override.start, override.end); - } - return override.end || 0; - } - - _getBinsCount (override) { - return override.bins || 0; - } - - _getWidth (override) { - let width = 0; - const binsCount = override.bins; - - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; - } - - return width; - } }; From d75ee965aeaba2fa1f1ff04d2b7e1c81a9979a9d Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 11 Sep 2017 11:48:33 +0200 Subject: [PATCH 065/177] changing some 'var' to 'let/const' --- lib/cartodb/models/dataview/histograms/date-histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 064de0ae..6060dbac 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -245,8 +245,8 @@ module.exports = class DateHistogram extends HistogramBase { return closer; } - var closerDiff = MAX_INTERVAL_VALUE - closer.value; - var currentDiff = MAX_INTERVAL_VALUE - current.value; + const closerDiff = MAX_INTERVAL_VALUE - closer.value; + const currentDiff = MAX_INTERVAL_VALUE - current.value; if (Number.isFinite(current.value) && closerDiff > currentDiff) { return current; @@ -332,7 +332,7 @@ module.exports = class DateHistogram extends HistogramBase { } _populateBinStart (override, firstRow) { - var binStart; + let binStart; if (firstRow.hasOwnProperty('timestamp')) { binStart = firstRow.timestamp; From 6c3fa045cd39a90f8a9d99ac3d37341bd5c62421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 13:53:05 +0200 Subject: [PATCH 066/177] Rename HistogramBase by BaseHistogram --- .../dataview/histograms/base-histogram.js | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 lib/cartodb/models/dataview/histograms/base-histogram.js diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js new file mode 100644 index 00000000..33a184d0 --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -0,0 +1,77 @@ +const BaseDataview = require('../base'); + +const TYPE = 'histogram'; + +module.exports = class BaseHistogram extends BaseDataview { + constructor (query, options, queries) { + super(); + + if (typeof options.column !== 'string') { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; + } + + sql (psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + if (this._columnType === null) { + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { + // assume numeric, will fail later + this._columnType = 'numeric'; + if (!err && !!type) { + this._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); + } + this.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + return this._buildQuery(psql, override, callback); + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); + } + + _getBinEnd (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.max(override.start, override.end); + } + return override.end || 0; + } + + _getBinsCount (override) { + return override.bins || 0; + } + + _getWidth (override) { + let width = 0; + const binsCount = override.bins; + + if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { + width = (override.end - override.start) / binsCount; + } + + return width; + } +}; \ No newline at end of file From ec23bfc79b29e7b1244d4be0b57eace3b12d7e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 13:54:46 +0200 Subject: [PATCH 067/177] Rename HistogramBase by BaseHistogram --- lib/cartodb/models/dataview/histogram.js | 14 ++-- .../dataview/histograms/date-histogram.js | 6 +- .../dataview/histograms/histogramBase.js | 77 ------------------- .../dataview/histograms/numeric-histogram.js | 4 +- 4 files changed, 12 insertions(+), 89 deletions(-) delete mode 100644 lib/cartodb/models/dataview/histograms/histogramBase.js diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7e0f71bf..6a59f021 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -8,13 +8,13 @@ const NUMERIC_HISTOGRAM = 'NumericHistogram'; module.exports = class Histogram { constructor (query, options, queries) { this.query = query; - this.options = options ||{}; + this.options = options || {}; this.queries = queries; - this.dataview = this._getHistogramImplemetation(); + this.dataview = this._getHistogramImplementation(); } - _getHistogramImplemetation (override) { + _getHistogramImplementation (override) { let implementation = null; switch (this._getHistogramSubtype(override)) { @@ -31,17 +31,17 @@ module.exports = class Histogram { } return implementation; - } - + } + _getHistogramSubtype (override = {}) { if(this.options.aggregation !== undefined || override.aggregation !== undefined) { return DATE_HISTOGRAM; } return NUMERIC_HISTOGRAM; } - + getResult (psql, override, callback) { - this.dataview = this._getHistogramImplemetation(override); + this.dataview = this._getHistogramImplementation(override); this.dataview.getResult(psql, override, callback); } diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 6060dbac..0ee832bc 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -1,4 +1,4 @@ -const HistogramBase = require('./histogramBase'); +const BaseHistogram = require('./base-histogram'); const dateIntervalQueryTpl = ctx => ` WITH @@ -165,10 +165,10 @@ const DATE_AGGREGATIONS = { } } */ -module.exports = class DateHistogram extends HistogramBase { +module.exports = class DateHistogram extends BaseHistogram { constructor (query, options, queries) { super(query, options, queries); - + this.aggregation = options.aggregation; this.offset = options.offset; } diff --git a/lib/cartodb/models/dataview/histograms/histogramBase.js b/lib/cartodb/models/dataview/histograms/histogramBase.js deleted file mode 100644 index dc6c40d6..00000000 --- a/lib/cartodb/models/dataview/histograms/histogramBase.js +++ /dev/null @@ -1,77 +0,0 @@ -const BaseDataview = require('../base'); - -const TYPE = 'histogram'; - -module.exports = class HistogramBase extends BaseDataview { - constructor (query, options, queries) { - super(); - - if (typeof options.column !== 'string') { - throw new Error('Histogram expects `column` in widget options'); - } - - this.query = query; - this.queries = queries; - this.column = options.column; - this.bins = options.bins; - - this._columnType = null; - } - - sql (psql, override, callback) { - if (!callback) { - callback = override; - override = {}; - } - - if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { - // assume numeric, will fail later - this._columnType = 'numeric'; - if (!err && !!type) { - this._columnType = Object.keys(type).find(function (key) { - return type[key]; - }); - } - this.sql(psql, override, callback); - }, true); // use read-only transaction - return null; - } - - return this._buildQuery(psql, override, callback); - } - - getType () { - return TYPE; - } - - toString () { - return JSON.stringify({ - _type: TYPE, - _column: this.column, - _query: this.query - }); - } - - _getBinEnd (override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.max(override.start, override.end); - } - return override.end || 0; - } - - _getBinsCount (override) { - return override.bins || 0; - } - - _getWidth (override) { - let width = 0; - const binsCount = override.bins; - - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; - } - - return width; - } -}; \ No newline at end of file diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index 2c6ac8e6..ece7babc 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -1,4 +1,4 @@ -const HistogramBase = require('./histogramBase'); +const BaseHistogram = require('./base-histogram'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; @@ -159,7 +159,7 @@ Numeric histogram: } } */ -module.exports = class NumericHistogram extends HistogramBase { +module.exports = class NumericHistogram extends BaseHistogram { constructor (query, options, queries) { super(query, options, queries); } From 4f33e0d794e5332eba9aa39e522b7d54baa19292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 15:34:42 +0200 Subject: [PATCH 068/177] Rename Histogram.dataview by Histogram.histogramImplementation --- lib/cartodb/models/dataview/histogram.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6a59f021..5099ee53 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -11,7 +11,7 @@ module.exports = class Histogram { this.options = options || {}; this.queries = queries; - this.dataview = this._getHistogramImplementation(); + this.histogramImplementation = this._getHistogramImplementation(); } _getHistogramImplementation (override) { @@ -41,8 +41,8 @@ module.exports = class Histogram { } getResult (psql, override, callback) { - this.dataview = this._getHistogramImplementation(override); - this.dataview.getResult(psql, override, callback); + this.histogramImplementation = this._getHistogramImplementation(override); + this.histogramImplementation.getResult(psql, override, callback); } // In order to keep previous behaviour with overviews, @@ -50,18 +50,18 @@ module.exports = class Histogram { // the concrete overview implementation sql (psql, override, callback) { - this.dataview.sql(psql, override, callback); + this.histogramImplementation.sql(psql, override, callback); } format (result, override) { - return this.dataview.format(result, override); + return this.histogramImplementation.format(result, override); } getType () { - return this.dataview.getType(); + return this.histogramImplementation.getType(); } toString () { - return this.dataview.toString(); + return this.histogramImplementation.toString(); } }; From 271932a80da90bdf74a83337306225d0cf6b0e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 17:17:42 +0200 Subject: [PATCH 069/177] Extract condition to a method --- lib/cartodb/models/dataview/histogram.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5099ee53..ba69c303 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -33,13 +33,18 @@ module.exports = class Histogram { return implementation; } - _getHistogramSubtype (override = {}) { - if(this.options.aggregation !== undefined || override.aggregation !== undefined) { + _getHistogramSubtype (override) { + if(this._isDateHistogram(override)) { return DATE_HISTOGRAM; } + return NUMERIC_HISTOGRAM; } + _isDateHistogram (override = {}) { + return (this.options.aggregation !== undefined || override.aggregation !== undefined); + } + getResult (psql, override, callback) { this.histogramImplementation = this._getHistogramImplementation(override); this.histogramImplementation.getResult(psql, override, callback); From 38e7e7132877555022e2f966589633a9be9e575b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 17:19:02 +0200 Subject: [PATCH 070/177] Implement template method pattern to format histogram query output --- .../dataview/histograms/base-histogram.js | 28 +++--- .../dataview/histograms/date-histogram.js | 99 +++++-------------- .../dataview/histograms/numeric-histogram.js | 72 +++++--------- 3 files changed, 61 insertions(+), 138 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js index 33a184d0..c9a53a88 100644 --- a/lib/cartodb/models/dataview/histograms/base-histogram.js +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -41,6 +41,12 @@ module.exports = class BaseHistogram extends BaseDataview { return this._buildQuery(psql, override, callback); } + format (result, override) { + const histogram = this._getSummary(result, override); + histogram.bins = this._getBuckets(result); + return histogram; + } + getType () { return TYPE; } @@ -53,25 +59,21 @@ module.exports = class BaseHistogram extends BaseDataview { }); } - _getBinEnd (override) { + _getBinStart (override) { + if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + return Math.min(override.start, override.end); + } + return override.start || 0; + } + + _getBinEnd (override = {}) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.max(override.start, override.end); } return override.end || 0; } - _getBinsCount (override) { + _getBinsCount (override = {}) { return override.bins || 0; } - - _getWidth (override) { - let width = 0; - const binsCount = override.bins; - - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; - } - - return width; - } }; \ No newline at end of file diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 0ee832bc..20523a46 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -259,90 +259,37 @@ module.exports = class DateHistogram extends BaseHistogram { }); } - format (result, override) { - override = override || {}; - let buckets = []; - - let binsCount = this._getBinsCount(override); - let width = this._getWidth(override); - let binsStart = this._getBinStart(override); - let nulls = 0; - let infinities = 0; - let nans = 0; - let avg; - let timestampStart; - let aggregation; - let offset; - - if (result.rows.length) { - const firstRow = result.rows[0]; - binsCount = firstRow.bins_number; - width = firstRow.bin_width || width; - avg = firstRow.avg_val; - nulls = firstRow.nulls_count; - timestampStart = firstRow.timestamp_start; - infinities = firstRow.infinities_count; - nans = firstRow.nans_count; - binsStart = this._populateBinStart(override, firstRow); - - aggregation = this._getAggregation(override); - offset = this._getOffset(override); - - buckets = result.rows.map(row => { - const { bin, min, max, avg, freq, timestamp } = row; - return { bin, min, max, avg, freq, timestamp }; - }); - } + _getSummary (result, override) { + const firstRow = result.rows[0] || {}; return { - aggregation: aggregation, - offset: offset, - timestamp_start: timestampStart, - bin_width: width, - bins_count: binsCount, - bins_start: binsStart, - nulls: nulls, - infinities: infinities, - nans: nans, - avg: avg, - bins: buckets + aggregation: this._getAggregation(override), + offset: this._getOffset(override), + timestamp_start: firstRow.timestamp_start, + + bin_width: firstRow.bin_width, + bins_count: firstRow.bins_number, + bins_start: firstRow.timestamp, + nulls: firstRow.nulls_count, + infinities: firstRow.infinities_count, + nans: firstRow.nans_count, + avg: firstRow.avg_val }; } - _getAggregation (override) { + _getBuckets (result) { + return result.rows.map(row => { + const { bin, min, max, avg, freq, timestamp } = row; + return { bin, min, max, avg, freq, timestamp }; + }); + } + + _getAggregation (override = {}) { return override && override.aggregation ? override.aggregation : this.aggregation; } - _getOffset (override) { - if (override && override.offset) { - return override.offset; - } - if (this.offset) { - return this.offset; - } - - return 0; - } - - _getBinStart (override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.min(override.start, override.end); - } - return override.start || 0; - } - - _populateBinStart (override, firstRow) { - let binStart; - - if (firstRow.hasOwnProperty('timestamp')) { - binStart = firstRow.timestamp; - } else if (override.hasOwnProperty('start')) { - binStart = this._getBinStart(override); - } else { - binStart = firstRow.min; - } - - return binStart; + _getOffset (override = {}) { + return override.offset || this.offset || 0; } _parseOffset (offset, aggregation) { diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index ece7babc..ed14d78f 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -182,14 +182,14 @@ module.exports = class NumericHistogram extends BaseHistogram { _buildQueryTpl (ctx) { return ` WITH - ${filteredQueryTpl(ctx)}, - ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._shouldOverrideRange(ctx._override) || this._shouldOverrideBins(ctx._override) ? - overrideBinsQueryTpl(ctx) : - `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` - }, - ${nullsQueryTpl(ctx)} - ${ctx._isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} + ${filteredQueryTpl(ctx)}, + ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._shouldOverrideRange(ctx._override) || this._shouldOverrideBins(ctx._override) ? + overrideBinsQueryTpl(ctx) : + `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` + }, + ${nullsQueryTpl(ctx)} + ${ctx._isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} ${histogramQueryTpl(ctx)} `; } @@ -206,54 +206,28 @@ module.exports = class NumericHistogram extends BaseHistogram { return override && override.hasOwnProperty('bins'); } - format (result, override) { - override = override || {}; - let buckets = []; - - let binsCount = this._getBinsCount(override); - let width = this._getWidth(override); - let binsStart = this._getBinStart(override); - let nulls = 0; - let infinities = 0; - let nans = 0; - let avg; - - if (result.rows.length) { - const firstRow = result.rows[0]; - binsCount = firstRow.bins_number; - width = firstRow.bin_width || width; - avg = firstRow.avg_val; - nulls = firstRow.nulls_count; - infinities = firstRow.infinities_count; - nans = firstRow.nans_count; - binsStart = this._populateBinStart(override, firstRow); - - buckets = result.rows.map(row => { - const { bin, min, max, avg, freq } = row; - return { bin, min, max, avg, freq }; - }); - } + _getSummary (result, override) { + const firstRow = result.rows[0] || {}; return { - bin_width: width, - bins_count: binsCount, - bins_start: binsStart, - nulls: nulls, - infinities: infinities, - nans: nans, - avg: avg, - bins: buckets + bin_width: firstRow.bin_width, + bins_count: firstRow.bins_number, + bins_start: this._populateBinStart(firstRow, override), + nulls: firstRow.nulls_count, + infinities: firstRow.infinities_count, + nans: firstRow.nans_count, + avg: firstRow.avg_val, }; } - _getBinStart (override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.min(override.start, override.end); - } - return override.start || 0; + _getBuckets (result) { + return result.rows.map(row => { + const { bin, min, max, avg, freq } = row; + return { bin, min, max, avg, freq }; + }); } - _populateBinStart (override, firstRow) { + _populateBinStart (firstRow, override = {}) { let binStart; if (override.hasOwnProperty('start')) { From fd9534797c519bf6be33cc9df07a62a3aa1abc4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 18:44:14 +0200 Subject: [PATCH 071/177] Minor refactors --- .../dataview/histograms/date-histogram.js | 32 ++++++++++--------- .../dataview/histograms/numeric-histogram.js | 3 ++ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 20523a46..a14e90e9 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -1,4 +1,5 @@ const BaseHistogram = require('./base-histogram'); +const debug = require('debug')('windshaft:dataview:date-histogram'); const dateIntervalQueryTpl = ctx => ` WITH @@ -186,16 +187,13 @@ module.exports = class DateHistogram extends BaseHistogram { } _buildQuery (psql, override, callback) { - const _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - const _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; - - if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + if (!DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override))) { return callback(new Error('Invalid aggregation value. Valid ones: ' + Object.keys(DATE_AGGREGATIONS).join(', ') )); } - if (_aggregation === 'auto') { + if (this._getAggregation(override) === 'auto') { this._getAutomaticAggregation(psql, function (err, aggregation) { if (err || aggregation === 'none') { this.aggregation = 'day'; @@ -212,12 +210,14 @@ module.exports = class DateHistogram extends BaseHistogram { _override: override, _query: this.query, _column: this.column, - _aggregation: _aggregation, + _aggregation: this._getAggregation(override), _start: this._getBinStart(override), _end: this._getBinEnd(override), - _offset: this._parseOffset(_offset, _aggregation) + _offset: this._parseOffset(override) }); + debug(histogramSql); + return callback(null, histogramSql); } @@ -285,22 +285,24 @@ module.exports = class DateHistogram extends BaseHistogram { } _getAggregation (override = {}) { - return override && override.aggregation ? override.aggregation : this.aggregation; + return override.aggregation ? override.aggregation : this.aggregation; } _getOffset (override = {}) { - return override.offset || this.offset || 0; + return Number.isFinite(override.offset) ? override.offset : (this.offset || 0); } - _parseOffset (offset, aggregation) { - if (!offset) { - return '0'; - } - if (aggregation === 'hour' || aggregation === 'minute') { + _parseOffset (override) { + if (this._shouldIgnoreOffset(override)) { return '0'; } - const offsetInHours = Math.ceil(offset / 3600); + const offsetInHours = Math.ceil(this._getOffset(override) / 3600); + return '' + offsetInHours; } + + _shouldIgnoreOffset (override) { + return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute'); + } }; diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index ed14d78f..60716eb6 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -1,4 +1,5 @@ const BaseHistogram = require('./base-histogram'); +const debug = require('debug')('windshaft:dataview:numeric-histogram'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; @@ -176,6 +177,8 @@ module.exports = class NumericHistogram extends BaseHistogram { _maxBins: BIN_MAX_NUMBER, }); + debug(histogramSql); + return callback(null, histogramSql); } From 09981c256053176ef5fd020724fc58bc7e124878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 18:57:16 +0200 Subject: [PATCH 072/177] Extract method to check valid aggregation --- lib/cartodb/models/dataview/histograms/date-histogram.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index a14e90e9..6695877a 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -187,7 +187,7 @@ module.exports = class DateHistogram extends BaseHistogram { } _buildQuery (psql, override, callback) { - if (!DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override))) { + if (!this.isValidAggregation(override)) { return callback(new Error('Invalid aggregation value. Valid ones: ' + Object.keys(DATE_AGGREGATIONS).join(', ') )); @@ -221,6 +221,10 @@ module.exports = class DateHistogram extends BaseHistogram { return callback(null, histogramSql); } + _isValidAggregation (override) { + return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override)); + } + _getAutomaticAggregation (psql, callback) { const dateIntervalQuery = dateIntervalQueryTpl({ query: this.query, From ee43378c68122c283e83020011ff4c7ce4565671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 18:57:50 +0200 Subject: [PATCH 073/177] Use arrow function --- .../models/dataview/histograms/date-histogram.js | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 6695877a..2684eec9 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -187,7 +187,7 @@ module.exports = class DateHistogram extends BaseHistogram { } _buildQuery (psql, override, callback) { - if (!this.isValidAggregation(override)) { + if (!this._isValidAggregation(override)) { return callback(new Error('Invalid aggregation value. Valid ones: ' + Object.keys(DATE_AGGREGATIONS).join(', ') )); @@ -238,13 +238,8 @@ module.exports = class DateHistogram extends BaseHistogram { const aggegations = result.rows[0]; const aggregation = Object.keys(aggegations) - .map(function (key) { - return { - name: key, - value: aggegations[key] - }; - }) - .reduce(function (closer, current) { + .map(key => ({ name: key, value: aggegations[key] })) + .reduce((closer, current) => { if (current.value > MAX_INTERVAL_VALUE) { return closer; } From 7c6924074846228dda9cada218b1e05423136a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 19:10:06 +0200 Subject: [PATCH 074/177] Use parameter default value --- lib/cartodb/models/dataview/histograms/base-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js index c9a53a88..2089e0c5 100644 --- a/lib/cartodb/models/dataview/histograms/base-histogram.js +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -59,7 +59,7 @@ module.exports = class BaseHistogram extends BaseDataview { }); } - _getBinStart (override) { + _getBinStart (override = {}) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); } From 5fc2b46d5684cde1611cabe7f5852e1c8fd1ef8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 19:14:11 +0200 Subject: [PATCH 075/177] Fix bad condition --- lib/cartodb/models/dataview/histograms/numeric-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index 60716eb6..c9a58736 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -187,7 +187,7 @@ module.exports = class NumericHistogram extends BaseHistogram { WITH ${filteredQueryTpl(ctx)}, ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._shouldOverrideRange(ctx._override) || this._shouldOverrideBins(ctx._override) ? + ${this._shouldOverrideBins(ctx._override) ? overrideBinsQueryTpl(ctx) : `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` }, From e3bffcd39d8c2d5eb86209ade7581c58581ba067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 19:24:01 +0200 Subject: [PATCH 076/177] Use inline functions to filter desired fields of the row --- lib/cartodb/models/dataview/histograms/date-histogram.js | 5 +---- lib/cartodb/models/dataview/histograms/numeric-histogram.js | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 2684eec9..631cb6c7 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -277,10 +277,7 @@ module.exports = class DateHistogram extends BaseHistogram { } _getBuckets (result) { - return result.rows.map(row => { - const { bin, min, max, avg, freq, timestamp } = row; - return { bin, min, max, avg, freq, timestamp }; - }); + return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp })); } _getAggregation (override = {}) { diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index c9a58736..fb158dc0 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -224,10 +224,7 @@ module.exports = class NumericHistogram extends BaseHistogram { } _getBuckets (result) { - return result.rows.map(row => { - const { bin, min, max, avg, freq } = row; - return { bin, min, max, avg, freq }; - }); + return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq })); } _populateBinStart (firstRow, override = {}) { From 9aa5a9e8503b8cf60605683f12648a95aff6c488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 19:26:28 +0200 Subject: [PATCH 077/177] Improve comment --- lib/cartodb/models/dataview/histograms/date-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 631cb6c7..fd7e8ba4 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -161,7 +161,7 @@ const DATE_AGGREGATIONS = { type: 'histogram', options: { column: 'date', // column data type: date - aggregation: 'day' // auto by default + aggregation: 'day' // MANDATORY offset: -7200 // OPTIONAL (UTC offset in seconds) } } From 03babcb43b397d2ffedf811af506074a4263d916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 19:31:38 +0200 Subject: [PATCH 078/177] Simplify condition and remove unused method --- lib/cartodb/models/dataview/histograms/numeric-histogram.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index fb158dc0..7a562ec7 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -186,7 +186,7 @@ module.exports = class NumericHistogram extends BaseHistogram { return ` WITH ${filteredQueryTpl(ctx)}, - ${this._shouldOverride(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._shouldOverrideRange(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, ${this._shouldOverrideBins(ctx._override) ? overrideBinsQueryTpl(ctx) : `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` @@ -197,10 +197,6 @@ module.exports = class NumericHistogram extends BaseHistogram { `; } - _shouldOverride (override) { - return this._shouldOverrideRange(override) && this._shouldOverrideBins(override); - } - _shouldOverrideRange (override) { return override && override.hasOwnProperty('start') && override.hasOwnProperty('end'); } From 3a936474cf9a1aa8ef07cbc701fdd5e9cd7954af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 11 Sep 2017 19:48:46 +0200 Subject: [PATCH 079/177] Fix bad merge with master --- lib/cartodb/models/dataview/histograms/date-histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index fd7e8ba4..e4176146 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -70,8 +70,8 @@ const dateBasicsQueryTpl = ctx => ` const dateOverrideBasicsQueryTpl = ctx => ` __cdb_basics AS ( SELECT - max(${ctx._end}) AS __cdb_max_val, - min(${ctx._start}) AS __cdb_min_val, + max(${ctx._end})::float AS __cdb_max_val, + min(${ctx._start})::float AS __cdb_min_val, avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, min( date_trunc( From 2576c3e7d50846829a27503e029430bc8ea707dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 12 Sep 2017 10:14:55 +0200 Subject: [PATCH 080/177] Rename _shouldOverrideRange & _shouldOverrideBins methods --- .../models/dataview/histograms/base-histogram.js | 10 ++++++++-- .../models/dataview/histograms/date-histogram.js | 4 +--- .../models/dataview/histograms/numeric-histogram.js | 10 +++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js index 2089e0c5..9b3f3759 100644 --- a/lib/cartodb/models/dataview/histograms/base-histogram.js +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -59,17 +59,23 @@ module.exports = class BaseHistogram extends BaseDataview { }); } + _hasOverridenRange (override) { + return override && override.hasOwnProperty('start') && override.hasOwnProperty('end'); + } + _getBinStart (override = {}) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + if (this._hasOverridenRange(override)) { return Math.min(override.start, override.end); } + return override.start || 0; } _getBinEnd (override = {}) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { + if (this._hasOverridenRange(override)) { return Math.max(override.start, override.end); } + return override.end || 0; } diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index e4176146..8124f2db 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -177,9 +177,7 @@ module.exports = class DateHistogram extends BaseHistogram { _buildQueryTpl (ctx) { return ` WITH - ${ctx._override && ctx._override.hasOwnProperty('start') && ctx._override.hasOwnProperty('end') ? - dateOverrideBasicsQueryTpl(ctx) : - dateBasicsQueryTpl(ctx)}, + ${this._hasOverridenRange(ctx._override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)}, ${dateBinsQueryTpl(ctx)}, ${nullsQueryTpl(ctx)} ${dateHistogramQueryTpl(ctx)} diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index 7a562ec7..638d3f0a 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -186,8 +186,8 @@ module.exports = class NumericHistogram extends BaseHistogram { return ` WITH ${filteredQueryTpl(ctx)}, - ${this._shouldOverrideRange(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._shouldOverrideBins(ctx._override) ? + ${this._hasOverridenRange(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._hasOverridenBins(ctx._override) ? overrideBinsQueryTpl(ctx) : `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` }, @@ -197,11 +197,7 @@ module.exports = class NumericHistogram extends BaseHistogram { `; } - _shouldOverrideRange (override) { - return override && override.hasOwnProperty('start') && override.hasOwnProperty('end'); - } - - _shouldOverrideBins (override) { + _hasOverridenBins (override) { return override && override.hasOwnProperty('bins'); } From 8f797c3c41f9fdf375e7170f889f9a655e42ba4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 12 Sep 2017 10:16:16 +0200 Subject: [PATCH 081/177] Fix EOF --- lib/cartodb/models/dataview/histograms/base-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js index 9b3f3759..ec251d09 100644 --- a/lib/cartodb/models/dataview/histograms/base-histogram.js +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -82,4 +82,4 @@ module.exports = class BaseHistogram extends BaseDataview { _getBinsCount (override = {}) { return override.bins || 0; } -}; \ No newline at end of file +}; From c0cfdad7d18788f1bcf2a099c8d8991dafa01c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 12 Sep 2017 10:38:53 +0200 Subject: [PATCH 082/177] Use hasOwnProperty method to check if histogram is a time-series --- lib/cartodb/models/dataview/histogram.js | 2 +- lib/cartodb/models/dataview/histograms/numeric-histogram.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index ba69c303..affbbd25 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -42,7 +42,7 @@ module.exports = class Histogram { } _isDateHistogram (override = {}) { - return (this.options.aggregation !== undefined || override.aggregation !== undefined); + return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation')); } getResult (psql, override, callback) { diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index 638d3f0a..1df88adb 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -64,7 +64,7 @@ const binsQueryTpl = ctx => ` ${ctx._maxBins} ) ) - END AS __cdb_bins_number + END AS __cdb_bins_number FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source LIMIT 1 ) From 4a89ad57d76ed6f123e2e3d08fa7ae36e54e1c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 12 Sep 2017 13:05:46 +0200 Subject: [PATCH 083/177] Remove '_' as prefix for template's context properties --- .../dataview/histograms/date-histogram.js | 64 +++++++-------- .../dataview/histograms/numeric-histogram.js | 78 +++++++++---------- 2 files changed, 71 insertions(+), 71 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 8124f2db..34a42f9a 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -45,45 +45,45 @@ const nullsQueryTpl = ctx => ` __cdb_nulls AS ( SELECT count(*) AS __cdb_nulls_count - FROM (${ctx._query}) __cdb_histogram_nulls - WHERE ${ctx._column} IS NULL + FROM (${ctx.query}) __cdb_histogram_nulls + WHERE ${ctx.column} IS NULL ) `; const dateBasicsQueryTpl = ctx => ` __cdb_basics AS ( SELECT - max(date_part('epoch', ${ctx._column})) AS __cdb_max_val, - min(date_part('epoch', ${ctx._column})) AS __cdb_min_val, - avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, + max(date_part('epoch', ${ctx.column})) AS __cdb_max_val, + min(date_part('epoch', ${ctx.column})) AS __cdb_min_val, + avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, min( date_trunc( - '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' + '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' ) ) AS __cdb_start_date, - max(${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}') AS __cdb_end_date, + max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date, count(1) AS __cdb_total_rows - FROM (${ctx._query}) __cdb_basics_query + FROM (${ctx.query}) __cdb_basics_query ) `; const dateOverrideBasicsQueryTpl = ctx => ` __cdb_basics AS ( SELECT - max(${ctx._end})::float AS __cdb_max_val, - min(${ctx._start})::float AS __cdb_min_val, - avg(date_part('epoch', ${ctx._column})) AS __cdb_avg_val, + max(${ctx.end})::float AS __cdb_max_val, + min(${ctx.start})::float AS __cdb_min_val, + avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, min( date_trunc( - '${ctx._aggregation}', - TO_TIMESTAMP(${ctx._start})::timestamp AT TIME ZONE '${ctx._offset}' + '${ctx.aggregation}', + TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}' ) ) AS __cdb_start_date, max( - TO_TIMESTAMP(${ctx._end})::timestamp AT TIME ZONE '${ctx._offset}' + TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}' ) AS __cdb_end_date, count(1) AS __cdb_total_rows - FROM (${ctx._query}) __cdb_basics_query + FROM (${ctx.query}) __cdb_basics_query ) `; @@ -98,7 +98,7 @@ const dateBinsQueryTpl = ctx => ` SELECT GENERATE_SERIES( __cdb_start_date::timestamptz, __cdb_end_date::timestamptz, - ${ctx._aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx._aggregation}'::interval`} + ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`} ) ) AS __cdb_bins_array FROM __cdb_basics @@ -117,7 +117,7 @@ const dateHistogramQueryTpl = ctx => ` 1, LEAST( WIDTH_BUCKET( - ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}', + ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}', __cdb_bins_array ), __cdb_bins_number @@ -128,17 +128,17 @@ const dateHistogramQueryTpl = ctx => ` date_part( 'epoch', date_trunc( - '${ctx._aggregation}', ${ctx._column}::timestamp AT TIME ZONE '${ctx._offset}' - ) AT TIME ZONE '${ctx._offset}' + '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' + ) AT TIME ZONE '${ctx.offset}' ) )::numeric AS timestamp, date_part('epoch', __cdb_start_date)::numeric AS timestamp_start, - min(date_part('epoch', ${ctx._column}))::numeric AS min, - max(date_part('epoch', ${ctx._column}))::numeric AS max, - avg(date_part('epoch', ${ctx._column}))::numeric AS avg, + min(date_part('epoch', ${ctx.column}))::numeric AS min, + max(date_part('epoch', ${ctx.column}))::numeric AS max, + avg(date_part('epoch', ${ctx.column}))::numeric AS avg, count(*) AS freq - FROM (${ctx._query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls - WHERE date_part('epoch', ${ctx._column}) IS NOT NULL + FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls + WHERE date_part('epoch', ${ctx.column}) IS NOT NULL GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start ORDER BY bin `; @@ -177,7 +177,7 @@ module.exports = class DateHistogram extends BaseHistogram { _buildQueryTpl (ctx) { return ` WITH - ${this._hasOverridenRange(ctx._override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)}, + ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)}, ${dateBinsQueryTpl(ctx)}, ${nullsQueryTpl(ctx)} ${dateHistogramQueryTpl(ctx)} @@ -205,13 +205,13 @@ module.exports = class DateHistogram extends BaseHistogram { } const histogramSql = this._buildQueryTpl({ - _override: override, - _query: this.query, - _column: this.column, - _aggregation: this._getAggregation(override), - _start: this._getBinStart(override), - _end: this._getBinEnd(override), - _offset: this._parseOffset(override) + override: override, + query: this.query, + column: this.column, + aggregation: this._getAggregation(override), + start: this._getBinStart(override), + end: this._getBinEnd(override), + offset: this._parseOffset(override) }); debug(histogramSql); diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index 1df88adb..ea191bea 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -4,27 +4,27 @@ const debug = require('debug')('windshaft:dataview:numeric-histogram'); const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; const filterOutSpecialNumericValues = ctx => ` - ${ctx._column} != 'infinity'::float + ${ctx.column} != 'infinity'::float AND - ${ctx._column} != '-infinity'::float + ${ctx.column} != '-infinity'::float AND - ${ctx._column} != 'NaN'::float + ${ctx.column} != 'NaN'::float `; const filteredQueryTpl = ctx => ` __cdb_filtered_source AS ( SELECT * - FROM (${ctx._query}) __cdb_filtered_source_query - WHERE ${ctx._column} IS NOT NULL - ${ctx._isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} + FROM (${ctx.query}) __cdb_filtered_source_query + WHERE ${ctx.column} IS NOT NULL + ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} ) `; const basicsQueryTpl = ctx => ` __cdb_basics AS ( SELECT - max(${ctx._column}) AS __cdb_max_val, min(${ctx._column}) AS __cdb_min_val, - avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val, + avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows FROM __cdb_filtered_source ) `; @@ -32,8 +32,8 @@ const basicsQueryTpl = ctx => ` const overrideBasicsQueryTpl = ctx => ` __cdb_basics AS ( SELECT - max(${ctx._end}) AS __cdb_max_val, min(${ctx._start}) AS __cdb_min_val, - avg(${ctx._column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val, + avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows FROM __cdb_filtered_source ) `; @@ -43,7 +43,7 @@ const iqrQueryTpl = ctx => ` SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr FROM ( SELECT quartile, max(_cdb_iqr_column) AS quartile_max from ( - SELECT ${ctx._column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx._column} + SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column} ) AS quartile FROM __cdb_filtered_source) _cdb_quartiles WHERE quartile = 1 or quartile = 3 @@ -58,10 +58,10 @@ const binsQueryTpl = ctx => ` CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 THEN 1 ELSE GREATEST( - LEAST(${ctx._minBins}, CAST(__cdb_total_rows AS INT)), + LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)), LEAST( CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT), - ${ctx._maxBins} + ${ctx.maxBins} ) ) END AS __cdb_bins_number @@ -72,7 +72,7 @@ const binsQueryTpl = ctx => ` const overrideBinsQueryTpl = ctx => ` __cdb_bins AS ( - SELECT ${ctx._override.bins} AS __cdb_bins_number + SELECT ${ctx.override.bins} AS __cdb_bins_number ) `; @@ -80,8 +80,8 @@ const nullsQueryTpl = ctx => ` __cdb_nulls AS ( SELECT count(*) AS __cdb_nulls_count - FROM (${ctx._query}) __cdb_histogram_nulls - WHERE ${ctx._column} IS NULL + FROM (${ctx.query}) __cdb_histogram_nulls + WHERE ${ctx.column} IS NULL ) `; @@ -89,11 +89,11 @@ const infinitiesQueryTpl = ctx => ` __cdb_infinities AS ( SELECT count(*) AS __cdb_infinities_count - FROM (${ctx._query}) __cdb_infinities_query + FROM (${ctx.query}) __cdb_infinities_query WHERE - ${ctx._column} = 'infinity'::float + ${ctx.column} = 'infinity'::float OR - ${ctx._column} = '-infinity'::float + ${ctx.column} = '-infinity'::float ) `; @@ -101,8 +101,8 @@ const nansQueryTpl = ctx => ` __cdb_nans AS ( SELECT count(*) AS __cdb_nans_count - FROM (${ctx._query}) __cdb_nans_query - WHERE ${ctx._column} = 'NaN'::float + FROM (${ctx.query}) __cdb_nans_query + WHERE ${ctx.column} = 'NaN'::float ) `; @@ -124,26 +124,26 @@ const histogramQueryTpl = ctx => ` (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, __cdb_bins_number AS bins_number, __cdb_nulls_count AS nulls_count, - ${ctx._isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} + ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} __cdb_avg_val AS avg_val, CASE WHEN __cdb_min_val = __cdb_max_val THEN 0 ELSE GREATEST( 1, LEAST( - WIDTH_BUCKET(${ctx._column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), + WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), __cdb_bins_number ) ) - 1 END AS bin, - min(${ctx._column})::numeric AS min, - max(${ctx._column})::numeric AS max, - avg(${ctx._column})::numeric AS avg, + min(${ctx.column})::numeric AS min, + max(${ctx.column})::numeric AS max, + avg(${ctx.column})::numeric AS avg, count(*) AS freq FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins - ${ctx._isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} + ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} GROUP BY bin, bins_number, bin_width, nulls_count, avg_val - ${ctx._isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} + ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} ORDER BY bin `; @@ -167,14 +167,14 @@ module.exports = class NumericHistogram extends BaseHistogram { _buildQuery (psql, override, callback) { const histogramSql = this._buildQueryTpl({ - _override: override, - _column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, - _isFloatColumn: this._columnType === 'float', - _query: this.query, - _start: this._getBinStart(override), - _end: this._getBinEnd(override), - _minBins: BIN_MIN_NUMBER, - _maxBins: BIN_MAX_NUMBER, + override: override, + column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, + isFloatColumn: this._columnType === 'float', + query: this.query, + start: this._getBinStart(override), + end: this._getBinEnd(override), + minBins: BIN_MIN_NUMBER, + maxBins: BIN_MAX_NUMBER, }); debug(histogramSql); @@ -186,13 +186,13 @@ module.exports = class NumericHistogram extends BaseHistogram { return ` WITH ${filteredQueryTpl(ctx)}, - ${this._hasOverridenRange(ctx._override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._hasOverridenBins(ctx._override) ? + ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._hasOverridenBins(ctx.override) ? overrideBinsQueryTpl(ctx) : `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` }, ${nullsQueryTpl(ctx)} - ${ctx._isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} + ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} ${histogramQueryTpl(ctx)} `; } From 6fd3388fa2de1ac9fd855ea102146fe9636f8969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 18:38:54 +0200 Subject: [PATCH 084/177] Replace dot templates by ES6 string templates --- lib/cartodb/models/dataview/aggregation.js | 252 ++++++++++++--------- 1 file changed, 149 insertions(+), 103 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 77f37723..1a4fc3ed 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -2,90 +2,134 @@ var _ = require('underscore'); var BaseWidget = require('./base'); var debug = require('debug')('windshaft:widget:aggregation'); -var dot = require('dot'); -dot.templateSettings.strip = false; +const filteredQueryTpl = ctx => ` + filtered_source AS ( + SELECT * + FROM (${ctx._query}) _cdb_filtered_source + ${ctx._aggregationColumn && ctx._isFloatColumn ? ` + WHERE + ${ctx._aggregationColumn} != 'infinity'::float + AND + ${ctx._aggregationColumn} != '-infinity'::float + AND + ${ctx._aggregationColumn} != 'NaN'::float` : + '' + } + ) +`; -var filteredQueryTpl = dot.template([ - 'filtered_source AS (', - ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source', - ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', - ' {{=it._aggregationColumn}} != \'infinity\'::float', - ' AND', - ' {{=it._aggregationColumn}} != \'-infinity\'::float', - ' AND', - ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', - ')' -].join(' \n')); +const summaryQueryTpl = ctx => ` + summary AS ( + SELECT + count(1) AS count, + sum(CASE WHEN ${ctx._column} IS NULL THEN 1 ELSE 0 END) AS nulls_count + ${ctx._isFloatColumn ? `, + sum( + CASE + WHEN ${ctx._aggregationColumn} = 'infinity'::float OR ${ctx._aggregationColumn} = '-infinity'::float + THEN 1 + ELSE 0 + END + ) AS infinities_count, + sum(CASE WHEN ${ctx._aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` : + '' + } + FROM (${ctx._query}) _cdb_aggregation_nulls + ) +`; -var summaryQueryTpl = dot.template([ - 'summary AS (', - ' SELECT', - ' count(1) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', - ' {{?it._isFloatColumn}},sum(', - ' CASE', - ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', - ' THEN 1', - ' ELSE 0', - ' END', - ' ) AS infinities_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', - ' FROM ({{=it._query}}) _cdb_aggregation_nulls', - ')' -].join('\n')); +const rankedCategoriesQueryTpl = ctx => ` + categories AS( + SELECT + ${ctx._column} AS category, + ${ctx._aggregation} AS value, + row_number() OVER (ORDER BY ${ctx._aggregation} desc) as rank + FROM filtered_source + ${ctx._aggregationColumn !== null ? `WHERE ${ctx._aggregationColumn} IS NOT NULL` : ''} + GROUP BY ${ctx._column} + ORDER BY 2 DESC + ) +`; -var rankedCategoriesQueryTpl = dot.template([ - 'categories AS(', - ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', - ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM filtered_source', - ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', - ' GROUP BY {{=it._column}}', - ' ORDER BY 2 DESC', - ')' -].join('\n')); +const categoriesSummaryMinMaxQueryTpl = () => ` + categories_summary_min_max AS( + SELECT + max(value) max_val, + min(value) min_val + FROM categories + ) +`; -var categoriesSummaryMinMaxQueryTpl = dot.template([ - 'categories_summary_min_max AS(', - ' SELECT max(value) max_val, min(value) min_val', - ' FROM categories', - ')' -].join('\n')); +const categoriesSummaryCountQueryTpl = ctx => ` + categories_summary_count AS( + SELECT count(1) AS categories_count + FROM ( + SELECT ${ctx._column} AS category + FROM (${ctx._query}) _cdb_categories + GROUP BY ${ctx._column} + ) _cdb_categories_count + ) +`; -var categoriesSummaryCountQueryTpl = dot.template([ - 'categories_summary_count AS(', - ' SELECT count(1) AS categories_count', - ' FROM (', - ' SELECT {{=it._column}} AS category', - ' FROM ({{=it._query}}) _cdb_categories', - ' GROUP BY {{=it._column}}', - ' ) _cdb_categories_count', - ')' -].join('\n')); +const specialNumericValuesColumns = () => `, nans_count, infinities_count`; -var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', - ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - ' FROM categories, summary, categories_summary_min_max, categories_summary_count', - ' WHERE rank < {{=it._limit}}', - 'UNION ALL', - 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', - ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - ' FROM categories, summary, categories_summary_min_max, categories_summary_count', - ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count,', - ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' -].join('\n')); +const rankedAggregationQueryTpl = ctx => ` + SELECT + CAST(category AS text), + value, + false as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM categories, summary, categories_summary_min_max, categories_summary_count + WHERE rank < ${ctx._limit} + UNION ALL + SELECT + 'Other' category, + ${ctx._aggregationFn}(value) as value, + true as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM categories, summary, categories_summary_min_max, categories_summary_count + WHERE rank >= ${ctx._limit} + GROUP BY + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } +`; -var aggregationQueryTpl = dot.template([ - 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count,', - ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - 'ORDER BY value DESC' -].join('\n')); +const aggregationQueryTpl = ctx => ` + SELECT + CAST(${ctx._column} AS text) AS category, + ${ctx._aggregation} AS value, + false as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM (${ctx._query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count + GROUP BY + category, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + ORDER BY value DESC +`; var CATEGORIES_LIMIT = 6; @@ -242,7 +286,8 @@ Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, ].join('\n'); }; -var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); +const aggregationFnQueryTpl = ctx => `${ctx._aggregationFn}(${ctx._aggregationColumn})`; + Aggregation.prototype.getAggregationSql = function() { return aggregationFnQueryTpl({ _aggregationFn: this.aggregation, @@ -290,32 +335,33 @@ Aggregation.prototype.format = function(result) { }; }; -var filterCategoriesQueryTpl = dot.template([ - 'SELECT {{=it._column}} AS category, {{=it._value}} AS value', - 'FROM ({{=it._query}}) _cdb_aggregation_search', - 'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}', - 'GROUP BY {{=it._column}}' -].join('\n')); - -var searchQueryTpl = dot.template([ - 'WITH', - 'search_unfiltered AS (', - ' {{=it._searchUnfiltered}}', - '),', - 'search_filtered AS (', - ' {{=it._searchFiltered}}', - '),', - 'search_union AS (', - ' SELECT * FROM search_unfiltered', - ' UNION ALL', - ' SELECT * FROM search_filtered', - ')', - 'SELECT category, sum(value) AS value', - 'FROM search_union', - 'GROUP BY category', - 'ORDER BY value desc' -].join('\n')); +const filterCategoriesQueryTpl = ctx => ` + SELECT + ${ctx._column} AS category, + ${ctx._value} AS value + FROM (${ctx._query}) _cdb_aggregation_search + WHERE CAST(${ctx._column} as text) ILIKE ${ctx._userQuery} + GROUP BY ${ctx._column} +`; +const searchQueryTpl = ctx => ` + WITH + search_unfiltered AS ( + ${ctx._searchUnfiltered} + ), + search_filtered AS ( + ${ctx._searchFiltered} + ), + search_union AS ( + SELECT * FROM search_unfiltered + UNION ALL + SELECT * FROM search_filtered + ) + SELECT category, sum(value) AS value + FROM search_union + GROUP BY category + ORDER BY value desc +`; Aggregation.prototype.search = function(psql, userQuery, callback) { var self = this; From 33e8657e35e090dbeedff19e06ba39bed746f06a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 18:40:09 +0200 Subject: [PATCH 085/177] Declare constants with const keyword --- lib/cartodb/models/dataview/aggregation.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 1a4fc3ed..8ba8621f 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -131,9 +131,9 @@ const aggregationQueryTpl = ctx => ` ORDER BY value DESC `; -var CATEGORIES_LIMIT = 6; +const CATEGORIES_LIMIT = 6; -var VALID_OPERATIONS = { +const VALID_OPERATIONS = { count: [], sum: ['aggregationColumn'], avg: ['aggregationColumn'], @@ -141,7 +141,7 @@ var VALID_OPERATIONS = { max: ['aggregationColumn'] }; -var TYPE = 'aggregation'; +const TYPE = 'aggregation'; /** { From 306df5be5a2d665ddf6500b65987c3712929de9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 19:16:08 +0200 Subject: [PATCH 086/177] Replace underscore's function by ES6 equivalents --- lib/cartodb/models/dataview/aggregation.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 8ba8621f..5cddac18 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -1,4 +1,3 @@ -var _ = require('underscore'); var BaseWidget = require('./base'); var debug = require('debug')('windshaft:widget:aggregation'); @@ -153,11 +152,13 @@ const TYPE = 'aggregation'; } */ function Aggregation(query, options, queries) { - if (!_.isString(options.column)) { + options = options || {}; + + if (typeof options.column !== 'string') { throw new Error('Aggregation expects `column` in widget options'); } - if (!_.isString(options.aggregation)) { + if (typeof options.aggregation !== 'string') { throw new Error('Aggregation expects `aggregation` operation in widget options'); } @@ -166,7 +167,8 @@ function Aggregation(query, options, queries) { } var requiredOptions = VALID_OPERATIONS[options.aggregation]; - var missingOptions = _.difference(requiredOptions, Object.keys(options)); + var missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + if (missingOptions.length > 0) { throw new Error( "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',') @@ -315,11 +317,7 @@ Aggregation.prototype.format = function(result) { minValue = firstRow.min_val; maxValue = firstRow.max_val; categoriesCount = firstRow.categories_count; - - result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', - 'max_val', 'categories_count', 'nans_count', 'infinities_count')); - }); + categories = result.rows.map(({ category, value, agg }) => ({ category, value, agg })); } return { From 4518b7cb6e7436d18acea77c9dcce8c98d49e59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 19:17:16 +0200 Subject: [PATCH 087/177] Declare requirement with const keyword --- lib/cartodb/models/dataview/aggregation.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 5cddac18..e8b431b4 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -1,5 +1,5 @@ -var BaseWidget = require('./base'); -var debug = require('debug')('windshaft:widget:aggregation'); +const BaseWidget = require('./base'); +const debug = require('debug')('windshaft:widget:aggregation'); const filteredQueryTpl = ctx => ` filtered_source AS ( From ee471184b91f59e04653a0c0ea3c4ca024135432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 19:19:25 +0200 Subject: [PATCH 088/177] Use default values for input params --- lib/cartodb/models/dataview/aggregation.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index e8b431b4..b2cfd212 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -151,9 +151,7 @@ const TYPE = 'aggregation'; } } */ -function Aggregation(query, options, queries) { - options = options || {}; - +function Aggregation(query, options = {}, queries = {}) { if (typeof options.column !== 'string') { throw new Error('Aggregation expects `column` in widget options'); } From 4d1a53c20f1abfef89cd554b2e4f4a13d7b76685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 19:27:25 +0200 Subject: [PATCH 089/177] Use string templates to build error message --- lib/cartodb/models/dataview/aggregation.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index b2cfd212..71394211 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -153,24 +153,22 @@ const TYPE = 'aggregation'; */ function Aggregation(query, options = {}, queries = {}) { if (typeof options.column !== 'string') { - throw new Error('Aggregation expects `column` in widget options'); + throw new Error(`Aggregation expects 'column' in widget options`); } if (typeof options.aggregation !== 'string') { - throw new Error('Aggregation expects `aggregation` operation in widget options'); + throw new Error(`Aggregation expects 'aggregation' operation in widget options`); } if (!VALID_OPERATIONS[options.aggregation]) { - throw new Error("Aggregation does not support '" + options.aggregation + "' operation"); + throw new Error(`Aggregation does not support '${options.aggregation}' operation`); } var requiredOptions = VALID_OPERATIONS[options.aggregation]; var missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); if (missingOptions.length > 0) { - throw new Error( - "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',') - ); + throw new Error(`Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`); } BaseWidget.apply(this); @@ -305,7 +303,6 @@ Aggregation.prototype.format = function(result) { var maxValue = 0; var categoriesCount = 0; - if (result.rows.length) { var firstRow = result.rows[0]; count = firstRow.count; From 75645e2d7af3dcc2b9ca79a6197381c40cad156d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 19:34:09 +0200 Subject: [PATCH 090/177] Use string templates to build categories CTE sql --- lib/cartodb/models/dataview/aggregation.js | 28 ++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 71394211..43dff8c0 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -251,37 +251,35 @@ Aggregation.prototype.sql = function(psql, override, callback) { }; Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { - return [ - "WITH", - [ - filteredQueryTpl({ + return ` + WITH + ${filteredQueryTpl({ _isFloatColumn: isFloatColumn, _query: this.query, _column: this.column, _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - summaryQueryTpl({ + })}, + ${summaryQueryTpl({ _isFloatColumn: isFloatColumn, _query: query, _column: column, _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - rankedCategoriesQueryTpl({ + })}, + ${rankedCategoriesQueryTpl({ _query: query, _column: column, _aggregation: this.getAggregationSql(), _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ + })}, + ${categoriesSummaryMinMaxQueryTpl({ _query: query, _column: column - }), - categoriesSummaryCountQueryTpl({ + })}, + ${categoriesSummaryCountQueryTpl({ _query: query, _column: column - }) - ].join(',\n') - ].join('\n'); + })} + `; }; const aggregationFnQueryTpl = ctx => `${ctx._aggregationFn}(${ctx._aggregationColumn})`; From 2f4f719f558f7e41701ffa713c9fc77e1c32cdbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Sep 2017 19:42:25 +0200 Subject: [PATCH 091/177] Use object properties in class method instead of passing them as parameters --- lib/cartodb/models/dataview/aggregation.js | 28 +++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 43dff8c0..ed0261ae 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -250,34 +250,34 @@ Aggregation.prototype.sql = function(psql, override, callback) { return callback(null, aggregationSql); }; -Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { +Aggregation.prototype.getCategoriesCTESql = function() { return ` WITH ${filteredQueryTpl({ - _isFloatColumn: isFloatColumn, + _isFloatColumn: this._isFloatColumn, _query: this.query, _column: this.column, - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null })}, ${summaryQueryTpl({ - _isFloatColumn: isFloatColumn, - _query: query, - _column: column, - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + _isFloatColumn: this._isFloatColumn, + _query: this.query, + _column: this.column, + _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null })}, ${rankedCategoriesQueryTpl({ - _query: query, - _column: column, + _query: this.query, + _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null })}, ${categoriesSummaryMinMaxQueryTpl({ - _query: query, - _column: column + _query: this.query, + _column: this.column })}, ${categoriesSummaryCountQueryTpl({ - _query: query, - _column: column + _query: this.query, + _column: this.column })} `; }; From a434015d5b3e020175dc38e248aece34c13b1484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 11:27:03 +0200 Subject: [PATCH 092/177] Move categories CTE template out of aggregation class --- lib/cartodb/models/dataview/aggregation.js | 111 ++++++++++----------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index ed0261ae..627eb73a 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -130,6 +130,41 @@ const aggregationQueryTpl = ctx => ` ORDER BY value DESC `; +const categoriesCTESqlTpl = ctx => ` + WITH + ${filteredQueryTpl({ + _isFloatColumn: ctx.isFloatColumn, + _query: ctx.query, + _column: ctx.column, + _aggregationColumn: ctx.aggregation !== 'count' ? ctx.aggregationColumn : null + })}, + ${summaryQueryTpl({ + _isFloatColumn: ctx.isFloatColumn, + _query: ctx.query, + _column: ctx.column, + _aggregationColumn: ctx.aggregation !== 'count' ? ctx.aggregationColumn : null + })}, + ${rankedCategoriesQueryTpl({ + _query: ctx.query, + _column: ctx.column, + _aggregation: aggregationFnQueryTpl({ + aggregation: ctx.aggregation, + aggregationColumn: ctx.aggregationColumn || 1 + }), + _aggregationColumn: ctx.aggregation !== 'count' ? ctx.aggregationColumn : null + })}, + ${categoriesSummaryMinMaxQueryTpl({ + _query: ctx.query, + _column: ctx.column + })}, + ${categoriesSummaryCountQueryTpl({ + _query: ctx.query, + _column: ctx.column + })} +`; + +const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`; + const CATEGORIES_LIMIT = 6; const VALID_OPERATIONS = { @@ -209,32 +244,37 @@ Aggregation.prototype.sql = function(psql, override, callback) { var aggregationSql; + + if (!!override.ownFilter) { aggregationSql = [ - this.getCategoriesCTESql( - _query, - this.column, - this.aggregation, - this.aggregationColumn, - this._isFloatColumn - ), + categoriesCTESqlTpl({ + query: _query, + column: this.column, + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn, + isFloatColumn: this._isFloatColumn + }), aggregationQueryTpl({ _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, - _aggregation: this.getAggregationSql(), + _aggregation: aggregationFnQueryTpl({ + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn || 1 + }), _limit: CATEGORIES_LIMIT }) ].join('\n'); } else { aggregationSql = [ - this.getCategoriesCTESql( - _query, - this.column, - this.aggregation, - this.aggregationColumn, - this._isFloatColumn - ), + categoriesCTESqlTpl({ + query: _query, + column: this.column, + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn, + isFloatColumn: this._isFloatColumn + }), rankedAggregationQueryTpl({ _isFloatColumn: this._isFloatColumn, _query: _query, @@ -250,47 +290,6 @@ Aggregation.prototype.sql = function(psql, override, callback) { return callback(null, aggregationSql); }; -Aggregation.prototype.getCategoriesCTESql = function() { - return ` - WITH - ${filteredQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.query, - _column: this.column, - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null - })}, - ${summaryQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.query, - _column: this.column, - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null - })}, - ${rankedCategoriesQueryTpl({ - _query: this.query, - _column: this.column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null - })}, - ${categoriesSummaryMinMaxQueryTpl({ - _query: this.query, - _column: this.column - })}, - ${categoriesSummaryCountQueryTpl({ - _query: this.query, - _column: this.column - })} - `; -}; - -const aggregationFnQueryTpl = ctx => `${ctx._aggregationFn}(${ctx._aggregationColumn})`; - -Aggregation.prototype.getAggregationSql = function() { - return aggregationFnQueryTpl({ - _aggregationFn: this.aggregation, - _aggregationColumn: this.aggregationColumn || 1 - }); -}; - Aggregation.prototype.format = function(result) { var categories = []; var count = 0; From 3682740f08ffd9fd43afd0df92003e08010c021f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 12:01:41 +0200 Subject: [PATCH 093/177] Build aggregation query with string templates avoiding to join all inner templates --- lib/cartodb/models/dataview/aggregation.js | 67 ++++++++-------------- 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 627eb73a..2ba36ede 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -240,50 +240,33 @@ Aggregation.prototype.sql = function(psql, override, callback) { return null; } - var _query = this.query; - - var aggregationSql; - - - - if (!!override.ownFilter) { - aggregationSql = [ - categoriesCTESqlTpl({ - query: _query, - column: this.column, + var aggregationSql = ` + ${categoriesCTESqlTpl({ + query: this.query, + column: this.column, + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn, + isFloatColumn: this._isFloatColumn + })} + ${!!override.ownFilter ? + `${aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, + _query: this.query, + _column: this.column, + _aggregation: aggregationFnQueryTpl({ aggregation: this.aggregation, - aggregationColumn: this.aggregationColumn, - isFloatColumn: this._isFloatColumn + aggregationColumn: this.aggregationColumn || 1 }), - aggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: _query, - _column: this.column, - _aggregation: aggregationFnQueryTpl({ - aggregation: this.aggregation, - aggregationColumn: this.aggregationColumn || 1 - }), - _limit: CATEGORIES_LIMIT - }) - ].join('\n'); - } else { - aggregationSql = [ - categoriesCTESqlTpl({ - query: _query, - column: this.column, - aggregation: this.aggregation, - aggregationColumn: this.aggregationColumn, - isFloatColumn: this._isFloatColumn - }), - rankedAggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: _query, - _column: this.column, - _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', - _limit: CATEGORIES_LIMIT - }) - ].join('\n'); - } + _limit: CATEGORIES_LIMIT + })}` : + `${rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, + _query: this.query, + _column: this.column, + _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', + _limit: CATEGORIES_LIMIT + })}`} + `; debug(aggregationSql); From 65dca454f497abf868e05876d2c9f059b27948f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 12:18:03 +0200 Subject: [PATCH 094/177] Move aggragation query to its own query template --- lib/cartodb/models/dataview/aggregation.js | 65 +++++++++++++--------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 2ba36ede..b22d2345 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -165,6 +165,36 @@ const categoriesCTESqlTpl = ctx => ` const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`; +const aggregationSqlTpl = ctx => ` + ${categoriesCTESqlTpl({ + query: ctx.query, + column: ctx.column, + aggregation: ctx.aggregation, + aggregationColumn: ctx.aggregationColumn, + isFloatColumn: ctx.isFloatColumn + })} + ${!!ctx.override.ownFilter ? + `${aggregationQueryTpl({ + _isFloatColumn: ctx.isFloatColumn, + _query: ctx.query, + _column: ctx.column, + _aggregation: aggregationFnQueryTpl({ + aggregation: ctx.aggregation, + aggregationColumn: ctx.aggregationColumn || 1 + }), + _limit: CATEGORIES_LIMIT + }) + }` : + `${rankedAggregationQueryTpl({ + _isFloatColumn: ctx.isFloatColumn, + _query: ctx.query, + _column: ctx.column, + _aggregationFn: ctx.aggregation !== 'count' ? ctx.aggregation : 'sum', + _limit: CATEGORIES_LIMIT + })} + `} +`; + const CATEGORIES_LIMIT = 6; const VALID_OPERATIONS = { @@ -240,33 +270,14 @@ Aggregation.prototype.sql = function(psql, override, callback) { return null; } - var aggregationSql = ` - ${categoriesCTESqlTpl({ - query: this.query, - column: this.column, - aggregation: this.aggregation, - aggregationColumn: this.aggregationColumn, - isFloatColumn: this._isFloatColumn - })} - ${!!override.ownFilter ? - `${aggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.query, - _column: this.column, - _aggregation: aggregationFnQueryTpl({ - aggregation: this.aggregation, - aggregationColumn: this.aggregationColumn || 1 - }), - _limit: CATEGORIES_LIMIT - })}` : - `${rankedAggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.query, - _column: this.column, - _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', - _limit: CATEGORIES_LIMIT - })}`} - `; + var aggregationSql = aggregationSqlTpl({ + override: override, + query: this.query, + column: this.column, + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn, + isFloatColumn: this._isFloatColumn + }); debug(aggregationSql); From b1508af007833d88459424efb1f1a445bff0d912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 13:14:12 +0200 Subject: [PATCH 095/177] Remove prefix to context's props in order to share it throught all templates avoiding to duplicate passing custom params again and again --- lib/cartodb/models/dataview/aggregation.js | 131 +++++++++------------ 1 file changed, 54 insertions(+), 77 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index b22d2345..be54ec7d 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -4,14 +4,14 @@ const debug = require('debug')('windshaft:widget:aggregation'); const filteredQueryTpl = ctx => ` filtered_source AS ( SELECT * - FROM (${ctx._query}) _cdb_filtered_source - ${ctx._aggregationColumn && ctx._isFloatColumn ? ` + FROM (${ctx.query}) _cdb_filtered_source + ${ctx.aggregationColumn && ctx.isFloatColumn ? ` WHERE - ${ctx._aggregationColumn} != 'infinity'::float + ${ctx.aggregationColumn} != 'infinity'::float AND - ${ctx._aggregationColumn} != '-infinity'::float + ${ctx.aggregationColumn} != '-infinity'::float AND - ${ctx._aggregationColumn} != 'NaN'::float` : + ${ctx.aggregationColumn} != 'NaN'::float` : '' } ) @@ -21,31 +21,31 @@ const summaryQueryTpl = ctx => ` summary AS ( SELECT count(1) AS count, - sum(CASE WHEN ${ctx._column} IS NULL THEN 1 ELSE 0 END) AS nulls_count - ${ctx._isFloatColumn ? `, + sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count + ${ctx.isFloatColumn ? `, sum( CASE - WHEN ${ctx._aggregationColumn} = 'infinity'::float OR ${ctx._aggregationColumn} = '-infinity'::float + WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float THEN 1 ELSE 0 END ) AS infinities_count, - sum(CASE WHEN ${ctx._aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` : + sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` : '' } - FROM (${ctx._query}) _cdb_aggregation_nulls + FROM (${ctx.query}) _cdb_aggregation_nulls ) `; const rankedCategoriesQueryTpl = ctx => ` categories AS( SELECT - ${ctx._column} AS category, - ${ctx._aggregation} AS value, - row_number() OVER (ORDER BY ${ctx._aggregation} desc) as rank + ${ctx.column} AS category, + ${ctx.aggregationFn} AS value, + row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank FROM filtered_source - ${ctx._aggregationColumn !== null ? `WHERE ${ctx._aggregationColumn} IS NOT NULL` : ''} - GROUP BY ${ctx._column} + ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''} + GROUP BY ${ctx.column} ORDER BY 2 DESC ) `; @@ -63,9 +63,9 @@ const categoriesSummaryCountQueryTpl = ctx => ` categories_summary_count AS( SELECT count(1) AS categories_count FROM ( - SELECT ${ctx._column} AS category - FROM (${ctx._query}) _cdb_categories - GROUP BY ${ctx._column} + SELECT ${ctx.column} AS category + FROM (${ctx.query}) _cdb_categories + GROUP BY ${ctx.column} ) _cdb_categories_count ) `; @@ -82,43 +82,43 @@ const rankedAggregationQueryTpl = ctx => ` max_val, count, categories_count - ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } FROM categories, summary, categories_summary_min_max, categories_summary_count - WHERE rank < ${ctx._limit} + WHERE rank < ${ctx.limit} UNION ALL SELECT 'Other' category, - ${ctx._aggregationFn}(value) as value, + ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value, true as agg, nulls_count, min_val, max_val, count, categories_count - ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } FROM categories, summary, categories_summary_min_max, categories_summary_count - WHERE rank >= ${ctx._limit} + WHERE rank >= ${ctx.limit} GROUP BY nulls_count, min_val, max_val, count, categories_count - ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } `; const aggregationQueryTpl = ctx => ` SELECT - CAST(${ctx._column} AS text) AS category, - ${ctx._aggregation} AS value, + CAST(${ctx.column} AS text) AS category, + ${ctx.aggregationFn} AS value, false as agg, nulls_count, min_val, max_val, count, categories_count - ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } - FROM (${ctx._query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count GROUP BY category, nulls_count, @@ -126,73 +126,45 @@ const aggregationQueryTpl = ctx => ` max_val, count, categories_count - ${ctx._isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } ORDER BY value DESC `; const categoriesCTESqlTpl = ctx => ` WITH ${filteredQueryTpl({ - _isFloatColumn: ctx.isFloatColumn, - _query: ctx.query, - _column: ctx.column, - _aggregationColumn: ctx.aggregation !== 'count' ? ctx.aggregationColumn : null + isFloatColumn: ctx.isFloatColumn, + query: ctx.query, + column: ctx.column, + aggregationColumn: ctx.aggregationColumn })}, ${summaryQueryTpl({ - _isFloatColumn: ctx.isFloatColumn, - _query: ctx.query, - _column: ctx.column, - _aggregationColumn: ctx.aggregation !== 'count' ? ctx.aggregationColumn : null + isFloatColumn: ctx.isFloatColumn, + query: ctx.query, + column: ctx.column, + aggregationColumn: ctx.aggregationColumn })}, ${rankedCategoriesQueryTpl({ - _query: ctx.query, - _column: ctx.column, - _aggregation: aggregationFnQueryTpl({ - aggregation: ctx.aggregation, - aggregationColumn: ctx.aggregationColumn || 1 - }), - _aggregationColumn: ctx.aggregation !== 'count' ? ctx.aggregationColumn : null + query: ctx.query, + column: ctx.column, + aggregationFn: ctx.aggregationFn, + aggregationColumn: ctx.aggregationColumn })}, ${categoriesSummaryMinMaxQueryTpl({ - _query: ctx.query, - _column: ctx.column + query: ctx.query, + column: ctx.column })}, ${categoriesSummaryCountQueryTpl({ - _query: ctx.query, - _column: ctx.column + query: ctx.query, + column: ctx.column })} `; const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`; const aggregationSqlTpl = ctx => ` - ${categoriesCTESqlTpl({ - query: ctx.query, - column: ctx.column, - aggregation: ctx.aggregation, - aggregationColumn: ctx.aggregationColumn, - isFloatColumn: ctx.isFloatColumn - })} - ${!!ctx.override.ownFilter ? - `${aggregationQueryTpl({ - _isFloatColumn: ctx.isFloatColumn, - _query: ctx.query, - _column: ctx.column, - _aggregation: aggregationFnQueryTpl({ - aggregation: ctx.aggregation, - aggregationColumn: ctx.aggregationColumn || 1 - }), - _limit: CATEGORIES_LIMIT - }) - }` : - `${rankedAggregationQueryTpl({ - _isFloatColumn: ctx.isFloatColumn, - _query: ctx.query, - _column: ctx.column, - _aggregationFn: ctx.aggregation !== 'count' ? ctx.aggregation : 'sum', - _limit: CATEGORIES_LIMIT - })} - `} + ${categoriesCTESqlTpl(ctx)} + ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`} `; const CATEGORIES_LIMIT = 6; @@ -275,8 +247,13 @@ Aggregation.prototype.sql = function(psql, override, callback) { query: this.query, column: this.column, aggregation: this.aggregation, - aggregationColumn: this.aggregationColumn, - isFloatColumn: this._isFloatColumn + aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null, + aggregationFn: aggregationFnQueryTpl({ + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn || 1 + }), + isFloatColumn: this._isFloatColumn, + limit: CATEGORIES_LIMIT }); debug(aggregationSql); From 132fce84c57f247d6ac30ebf92fc9ebb7d169021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 15:07:30 +0200 Subject: [PATCH 096/177] Remove string template in the middle --- lib/cartodb/models/dataview/aggregation.js | 41 +++++----------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index be54ec7d..6f87aba1 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -130,40 +130,15 @@ const aggregationQueryTpl = ctx => ` ORDER BY value DESC `; -const categoriesCTESqlTpl = ctx => ` - WITH - ${filteredQueryTpl({ - isFloatColumn: ctx.isFloatColumn, - query: ctx.query, - column: ctx.column, - aggregationColumn: ctx.aggregationColumn - })}, - ${summaryQueryTpl({ - isFloatColumn: ctx.isFloatColumn, - query: ctx.query, - column: ctx.column, - aggregationColumn: ctx.aggregationColumn - })}, - ${rankedCategoriesQueryTpl({ - query: ctx.query, - column: ctx.column, - aggregationFn: ctx.aggregationFn, - aggregationColumn: ctx.aggregationColumn - })}, - ${categoriesSummaryMinMaxQueryTpl({ - query: ctx.query, - column: ctx.column - })}, - ${categoriesSummaryCountQueryTpl({ - query: ctx.query, - column: ctx.column - })} -`; - const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`; -const aggregationSqlTpl = ctx => ` - ${categoriesCTESqlTpl(ctx)} +const aggregationDataviewQueryTpl = ctx => ` + WITH + ${filteredQueryTpl(ctx)}, + ${summaryQueryTpl(ctx)}, + ${rankedCategoriesQueryTpl(ctx)}, + ${categoriesSummaryMinMaxQueryTpl(ctx)}, + ${categoriesSummaryCountQueryTpl(ctx)} ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`} `; @@ -242,7 +217,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { return null; } - var aggregationSql = aggregationSqlTpl({ + var aggregationSql = aggregationDataviewQueryTpl({ override: override, query: this.query, column: this.column, From e2ab48bee2630050a8636776ebe069238c4fe398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 15:22:58 +0200 Subject: [PATCH 097/177] Remove prefix to context's properties --- lib/cartodb/models/dataview/aggregation.js | 40 ++++++++++++---------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 6f87aba1..575c7202 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -273,20 +273,20 @@ Aggregation.prototype.format = function(result) { const filterCategoriesQueryTpl = ctx => ` SELECT - ${ctx._column} AS category, - ${ctx._value} AS value - FROM (${ctx._query}) _cdb_aggregation_search - WHERE CAST(${ctx._column} as text) ILIKE ${ctx._userQuery} - GROUP BY ${ctx._column} + ${ctx.column} AS category, + ${ctx.value} AS value + FROM (${ctx.query}) _cdb_aggregation_search + WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery} + GROUP BY ${ctx.column} `; const searchQueryTpl = ctx => ` WITH search_unfiltered AS ( - ${ctx._searchUnfiltered} + ${ctx.searchUnfiltered} ), search_filtered AS ( - ${ctx._searchFiltered} + ${ctx.searchFiltered} ), search_union AS ( SELECT * FROM search_unfiltered @@ -302,26 +302,28 @@ const searchQueryTpl = ctx => ` Aggregation.prototype.search = function(psql, userQuery, callback) { var self = this; - var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); - var _value = this.aggregation !== 'count' && this.aggregationColumn ? + var escapedUserQuery = psql.escapeLiteral('%' + userQuery + '%'); + var value = this.aggregation !== 'count' && this.aggregationColumn ? this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ - _searchUnfiltered: filterCategoriesQueryTpl({ - _query: this.query, - _column: this.column, - _value: '0', - _userQuery: _userQuery + searchUnfiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: '0', + userQuery: escapedUserQuery }), - _searchFiltered: filterCategoriesQueryTpl({ - _query: this.query, - _column: this.column, - _value: _value, - _userQuery: _userQuery + searchFiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: value, + userQuery: escapedUserQuery }) }); + debug(query); + psql.query(query, function(err, result) { if (err) { return callback(err, result); From 0a894da0df1cf6da599e85623500ab6cda5b099b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 15:27:57 +0200 Subject: [PATCH 098/177] Remove prefix to context's properties --- lib/cartodb/models/dataview/aggregation.js | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 575c7202..ecf9419c 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -283,10 +283,20 @@ const filterCategoriesQueryTpl = ctx => ` const searchQueryTpl = ctx => ` WITH search_unfiltered AS ( - ${ctx.searchUnfiltered} + ${filterCategoriesQueryTpl({ + query: ctx.query, + column: ctx.column, + value: '0', + userQuery: escapedUserQuery + })} ), search_filtered AS ( - ${ctx.searchFiltered} + ${filterCategoriesQueryTpl({ + query: ctx.query, + column: ctx.column, + value: value, + userQuery: escapedUserQuery + })} ), search_union AS ( SELECT * FROM search_unfiltered @@ -308,18 +318,10 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ - searchUnfiltered: filterCategoriesQueryTpl({ - query: this.query, - column: this.column, - value: '0', - userQuery: escapedUserQuery - }), - searchFiltered: filterCategoriesQueryTpl({ - query: this.query, - column: this.column, - value: value, - userQuery: escapedUserQuery - }) + query: this.query, + column: this.column, + value: value, + userQuery: escapedUserQuery }); debug(query); From 02949003a96b6eaec2bee1257bd8f26353e49c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 15:48:54 +0200 Subject: [PATCH 099/177] Build search query in two steps --- lib/cartodb/models/dataview/aggregation.js | 30 ++++++++++------------ 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index ecf9419c..575c7202 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -283,20 +283,10 @@ const filterCategoriesQueryTpl = ctx => ` const searchQueryTpl = ctx => ` WITH search_unfiltered AS ( - ${filterCategoriesQueryTpl({ - query: ctx.query, - column: ctx.column, - value: '0', - userQuery: escapedUserQuery - })} + ${ctx.searchUnfiltered} ), search_filtered AS ( - ${filterCategoriesQueryTpl({ - query: ctx.query, - column: ctx.column, - value: value, - userQuery: escapedUserQuery - })} + ${ctx.searchFiltered} ), search_union AS ( SELECT * FROM search_unfiltered @@ -318,10 +308,18 @@ Aggregation.prototype.search = function(psql, userQuery, callback) { // TODO unfiltered will be wrong as filters are already applied at this point var query = searchQueryTpl({ - query: this.query, - column: this.column, - value: value, - userQuery: escapedUserQuery + searchUnfiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: '0', + userQuery: escapedUserQuery + }), + searchFiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: value, + userQuery: escapedUserQuery + }) }); debug(query); From 086eff01a907e7dc69039a47e51075914ee961f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 15:57:18 +0200 Subject: [PATCH 100/177] Use ES6 class syntax --- lib/cartodb/models/dataview/aggregation.js | 285 ++++++++++----------- 1 file changed, 142 insertions(+), 143 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 575c7202..039cbea0 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -163,112 +163,158 @@ const TYPE = 'aggregation'; } } */ -function Aggregation(query, options = {}, queries = {}) { - if (typeof options.column !== 'string') { - throw new Error(`Aggregation expects 'column' in widget options`); +module.exports = class Aggregation extends BaseWidget { + constructor (query, options = {}, queries = {}) { + super(); + + if (typeof options.column !== 'string') { + throw new Error(`Aggregation expects 'column' in widget options`); + } + + if (typeof options.aggregation !== 'string') { + throw new Error(`Aggregation expects 'aggregation' operation in widget options`); + } + + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error(`Aggregation does not support '${options.aggregation}' operation`); + } + + var requiredOptions = VALID_OPERATIONS[options.aggregation]; + var missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + + if (missingOptions.length > 0) { + throw new Error( + `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}` + ); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } - if (typeof options.aggregation !== 'string') { - throw new Error(`Aggregation expects 'aggregation' operation in widget options`); - } + sql (psql, override, callback) { + var self = this; - if (!VALID_OPERATIONS[options.aggregation]) { - throw new Error(`Aggregation does not support '${options.aggregation}' operation`); - } + if (!callback) { + callback = override; + override = {}; + } - var requiredOptions = VALID_OPERATIONS[options.aggregation]; - var missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } - if (missingOptions.length > 0) { - throw new Error(`Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`); - } - - BaseWidget.apply(this); - - this.query = query; - this.queries = queries; - this.column = options.column; - this.aggregation = options.aggregation; - this.aggregationColumn = options.aggregationColumn; - this._isFloatColumn = null; -} - -Aggregation.prototype = new BaseWidget(); -Aggregation.prototype.constructor = Aggregation; - -module.exports = Aggregation; - -Aggregation.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; - } - - if (this.aggregationColumn && this._isFloatColumn === null) { - this._isFloatColumn = false; - this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { - if (!err && !!type) { - self._isFloatColumn = type.float; - } - self.sql(psql, override, callback); - }); - return null; - } - - var aggregationSql = aggregationDataviewQueryTpl({ - override: override, - query: this.query, - column: this.column, - aggregation: this.aggregation, - aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null, - aggregationFn: aggregationFnQueryTpl({ + var aggregationSql = aggregationDataviewQueryTpl({ + override: override, + query: this.query, + column: this.column, aggregation: this.aggregation, - aggregationColumn: this.aggregationColumn || 1 - }), - isFloatColumn: this._isFloatColumn, - limit: CATEGORIES_LIMIT - }); + aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null, + aggregationFn: aggregationFnQueryTpl({ + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn || 1 + }), + isFloatColumn: this._isFloatColumn, + limit: CATEGORIES_LIMIT + }); - debug(aggregationSql); + debug(aggregationSql); - return callback(null, aggregationSql); -}; - -Aggregation.prototype.format = function(result) { - var categories = []; - var count = 0; - var nulls = 0; - var nans = 0; - var infinities = 0; - var minValue = 0; - var maxValue = 0; - var categoriesCount = 0; - - if (result.rows.length) { - var firstRow = result.rows[0]; - count = firstRow.count; - nulls = firstRow.nulls_count; - nans = firstRow.nans_count; - infinities = firstRow.infinities_count; - minValue = firstRow.min_val; - maxValue = firstRow.max_val; - categoriesCount = firstRow.categories_count; - categories = result.rows.map(({ category, value, agg }) => ({ category, value, agg })); + return callback(null, aggregationSql); } - return { - aggregation: this.aggregation, - count: count, - nulls: nulls, - nans: nans, - infinities: infinities, - min: minValue, - max: maxValue, - categoriesCount: categoriesCount, - categories: categories - }; + format (result) { + var categories = []; + var count = 0; + var nulls = 0; + var nans = 0; + var infinities = 0; + var minValue = 0; + var maxValue = 0; + var categoriesCount = 0; + + if (result.rows.length) { + var firstRow = result.rows[0]; + count = firstRow.count; + nulls = firstRow.nulls_count; + nans = firstRow.nans_count; + infinities = firstRow.infinities_count; + minValue = firstRow.min_val; + maxValue = firstRow.max_val; + categoriesCount = firstRow.categories_count; + categories = result.rows.map(({ category, value, agg }) => ({ category, value, agg })); + } + + return { + aggregation: this.aggregation, + count: count, + nulls: nulls, + nans: nans, + infinities: infinities, + min: minValue, + max: maxValue, + categoriesCount: categoriesCount, + categories: categories + }; + } + + search (psql, userQuery, callback) { + var self = this; + + var escapedUserQuery = psql.escapeLiteral('%' + userQuery + '%'); + var value = this.aggregation !== 'count' && this.aggregationColumn ? + this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; + + // TODO unfiltered will be wrong as filters are already applied at this point + var query = searchQueryTpl({ + searchUnfiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: '0', + userQuery: escapedUserQuery + }), + searchFiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: value, + userQuery: escapedUserQuery + }) + }); + + debug(query); + + psql.query(query, function(err, result) { + if (err) { + return callback(err, result); + } + + return callback(null, {type: self.getType(), categories: result.rows }); + }, true); // use read-only transaction + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _aggregation: this.aggregation + }); + } }; const filterCategoriesQueryTpl = ctx => ` @@ -298,50 +344,3 @@ const searchQueryTpl = ctx => ` GROUP BY category ORDER BY value desc `; - -Aggregation.prototype.search = function(psql, userQuery, callback) { - var self = this; - - var escapedUserQuery = psql.escapeLiteral('%' + userQuery + '%'); - var value = this.aggregation !== 'count' && this.aggregationColumn ? - this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; - - // TODO unfiltered will be wrong as filters are already applied at this point - var query = searchQueryTpl({ - searchUnfiltered: filterCategoriesQueryTpl({ - query: this.query, - column: this.column, - value: '0', - userQuery: escapedUserQuery - }), - searchFiltered: filterCategoriesQueryTpl({ - query: this.query, - column: this.column, - value: value, - userQuery: escapedUserQuery - }) - }); - - debug(query); - - psql.query(query, function(err, result) { - if (err) { - return callback(err, result); - } - - return callback(null, {type: self.getType(), categories: result.rows }); - }, true); // use read-only transaction -}; - -Aggregation.prototype.getType = function() { - return TYPE; -}; - -Aggregation.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _column: this.column, - _aggregation: this.aggregation - }); -}; From 46254eaf743f7851878a0a7baef52a342379b342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 15:59:15 +0200 Subject: [PATCH 101/177] rename BaseWidget by BaseDataview --- lib/cartodb/models/dataview/aggregation.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 039cbea0..f0ea5608 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -1,4 +1,4 @@ -const BaseWidget = require('./base'); +const BaseDataview = require('./base'); const debug = require('debug')('windshaft:widget:aggregation'); const filteredQueryTpl = ctx => ` @@ -163,7 +163,7 @@ const TYPE = 'aggregation'; } } */ -module.exports = class Aggregation extends BaseWidget { +module.exports = class Aggregation extends BaseDataview { constructor (query, options = {}, queries = {}) { super(); @@ -176,7 +176,7 @@ module.exports = class Aggregation extends BaseWidget { } if (!VALID_OPERATIONS[options.aggregation]) { - throw new Error(`Aggregation does not support '${options.aggregation}' operation`); + throw new Error(`Aggregation does not support '${options.aggregation}' operation`); } var requiredOptions = VALID_OPERATIONS[options.aggregation]; From cd13107a4dbefada5ac6e0fdbf27346261c18cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:08:12 +0200 Subject: [PATCH 102/177] Use ES6 let & const to declare variables --- lib/cartodb/models/dataview/aggregation.js | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index f0ea5608..b1e56f28 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -179,8 +179,8 @@ module.exports = class Aggregation extends BaseDataview { throw new Error(`Aggregation does not support '${options.aggregation}' operation`); } - var requiredOptions = VALID_OPERATIONS[options.aggregation]; - var missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + const requiredOptions = VALID_OPERATIONS[options.aggregation]; + const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); if (missingOptions.length > 0) { throw new Error( @@ -197,7 +197,7 @@ module.exports = class Aggregation extends BaseDataview { } sql (psql, override, callback) { - var self = this; + const self = this; if (!callback) { callback = override; @@ -215,7 +215,7 @@ module.exports = class Aggregation extends BaseDataview { return null; } - var aggregationSql = aggregationDataviewQueryTpl({ + const aggregationSql = aggregationDataviewQueryTpl({ override: override, query: this.query, column: this.column, @@ -235,17 +235,17 @@ module.exports = class Aggregation extends BaseDataview { } format (result) { - var categories = []; - var count = 0; - var nulls = 0; - var nans = 0; - var infinities = 0; - var minValue = 0; - var maxValue = 0; - var categoriesCount = 0; + let categories = []; + let count = 0; + let nulls = 0; + let nans = 0; + let infinities = 0; + let minValue = 0; + let maxValue = 0; + let categoriesCount = 0; if (result.rows.length) { - var firstRow = result.rows[0]; + const firstRow = result.rows[0]; count = firstRow.count; nulls = firstRow.nulls_count; nans = firstRow.nans_count; @@ -270,14 +270,14 @@ module.exports = class Aggregation extends BaseDataview { } search (psql, userQuery, callback) { - var self = this; + const self = this; - var escapedUserQuery = psql.escapeLiteral('%' + userQuery + '%'); - var value = this.aggregation !== 'count' && this.aggregationColumn ? + const escapedUserQuery = psql.escapeLiteral('%' + userQuery + '%'); + const value = this.aggregation !== 'count' && this.aggregationColumn ? this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point - var query = searchQueryTpl({ + const query = searchQueryTpl({ searchUnfiltered: filterCategoriesQueryTpl({ query: this.query, column: this.column, From b05083bcfc09ebf6b1326bcb394905a8c46ef196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:09:44 +0200 Subject: [PATCH 103/177] Move search's templates along aggregation's templates --- lib/cartodb/models/dataview/aggregation.js | 56 +++++++++++----------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index b1e56f28..d51edf04 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -142,6 +142,34 @@ const aggregationDataviewQueryTpl = ctx => ` ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`} `; +const filterCategoriesQueryTpl = ctx => ` + SELECT + ${ctx.column} AS category, + ${ctx.value} AS value + FROM (${ctx.query}) _cdb_aggregation_search + WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery} + GROUP BY ${ctx.column} +`; + +const searchQueryTpl = ctx => ` + WITH + search_unfiltered AS ( + ${ctx.searchUnfiltered} + ), + search_filtered AS ( + ${ctx.searchFiltered} + ), + search_union AS ( + SELECT * FROM search_unfiltered + UNION ALL + SELECT * FROM search_filtered + ) + SELECT category, sum(value) AS value + FROM search_union + GROUP BY category + ORDER BY value desc +`; + const CATEGORIES_LIMIT = 6; const VALID_OPERATIONS = { @@ -316,31 +344,3 @@ module.exports = class Aggregation extends BaseDataview { }); } }; - -const filterCategoriesQueryTpl = ctx => ` - SELECT - ${ctx.column} AS category, - ${ctx.value} AS value - FROM (${ctx.query}) _cdb_aggregation_search - WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery} - GROUP BY ${ctx.column} -`; - -const searchQueryTpl = ctx => ` - WITH - search_unfiltered AS ( - ${ctx.searchUnfiltered} - ), - search_filtered AS ( - ${ctx.searchFiltered} - ), - search_union AS ( - SELECT * FROM search_unfiltered - UNION ALL - SELECT * FROM search_filtered - ) - SELECT category, sum(value) AS value - FROM search_union - GROUP BY category - ORDER BY value desc -`; From 746292610aeec58b901e569a9d2110c2c0fa6ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:10:23 +0200 Subject: [PATCH 104/177] Rename debug namespace --- lib/cartodb/models/dataview/aggregation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index d51edf04..b6f54263 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -1,5 +1,5 @@ const BaseDataview = require('./base'); -const debug = require('debug')('windshaft:widget:aggregation'); +const debug = require('debug')('windshaft:dataview:aggregation'); const filteredQueryTpl = ctx => ` filtered_source AS ( From cc2cf78264bcc6459ec8610ff12621ec90823ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:12:39 +0200 Subject: [PATCH 105/177] Rename all 'widget' ocurrences by 'dataview' --- lib/cartodb/models/dataview/aggregation.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index b6f54263..80f77062 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -196,11 +196,11 @@ module.exports = class Aggregation extends BaseDataview { super(); if (typeof options.column !== 'string') { - throw new Error(`Aggregation expects 'column' in widget options`); + throw new Error(`Aggregation expects 'column' in dataview options`); } if (typeof options.aggregation !== 'string') { - throw new Error(`Aggregation expects 'aggregation' operation in widget options`); + throw new Error(`Aggregation expects 'aggregation' operation in dataview options`); } if (!VALID_OPERATIONS[options.aggregation]) { From c2e99219effef22af3bdc2c3b0718e6f9c8045cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:30:46 +0200 Subject: [PATCH 106/177] Use ES6 goodies to refactor format method --- lib/cartodb/models/dataview/aggregation.js | 43 ++++++++-------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 80f77062..e60697cb 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -263,37 +263,26 @@ module.exports = class Aggregation extends BaseDataview { } format (result) { - let categories = []; - let count = 0; - let nulls = 0; - let nans = 0; - let infinities = 0; - let minValue = 0; - let maxValue = 0; - let categoriesCount = 0; - - if (result.rows.length) { - const firstRow = result.rows[0]; - count = firstRow.count; - nulls = firstRow.nulls_count; - nans = firstRow.nans_count; - infinities = firstRow.infinities_count; - minValue = firstRow.min_val; - maxValue = firstRow.max_val; - categoriesCount = firstRow.categories_count; - categories = result.rows.map(({ category, value, agg }) => ({ category, value, agg })); - } + const { + count = 0, + nulls_count = 0, + nans_count = 0, + infinities_count = 0, + min_val = 0, + max_val = 0, + categories_count = 0 + } = result.rows[0] || {}; return { aggregation: this.aggregation, count: count, - nulls: nulls, - nans: nans, - infinities: infinities, - min: minValue, - max: maxValue, - categoriesCount: categoriesCount, - categories: categories + nulls: nulls_count, + nans: nans_count, + infinities: infinities_count, + min: min_val, + max: max_val, + categoriesCount: categories_count, + categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg })) }; } From 46860541fe035c5627b8576741bcf3bf721d1977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:42:25 +0200 Subject: [PATCH 107/177] Apply extract method to validate input options --- lib/cartodb/models/dataview/aggregation.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index e60697cb..c7626736 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -195,6 +195,17 @@ module.exports = class Aggregation extends BaseDataview { constructor (query, options = {}, queries = {}) { super(); + this._checkOptions(options); + + this.query = query; + this.queries = queries; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; + } + + _checkOptions () { if (typeof options.column !== 'string') { throw new Error(`Aggregation expects 'column' in dataview options`); } @@ -215,13 +226,6 @@ module.exports = class Aggregation extends BaseDataview { `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}` ); } - - this.query = query; - this.queries = queries; - this.column = options.column; - this.aggregation = options.aggregation; - this.aggregationColumn = options.aggregationColumn; - this._isFloatColumn = null; } sql (psql, override, callback) { From 38e2c040d175a4171ed469fb9f2fc210d93da5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:45:45 +0200 Subject: [PATCH 108/177] Use template string to escape literals --- lib/cartodb/models/dataview/aggregation.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index c7626736..29eebe06 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -292,8 +292,7 @@ module.exports = class Aggregation extends BaseDataview { search (psql, userQuery, callback) { const self = this; - - const escapedUserQuery = psql.escapeLiteral('%' + userQuery + '%'); + const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`); const value = this.aggregation !== 'count' && this.aggregationColumn ? this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; From 1f3b0beddfa5120d56a7337a0fd5c3de23a1f3f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:47:05 +0200 Subject: [PATCH 109/177] Fix missing parameter --- lib/cartodb/models/dataview/aggregation.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 29eebe06..f621e536 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -205,7 +205,7 @@ module.exports = class Aggregation extends BaseDataview { this._isFloatColumn = null; } - _checkOptions () { + _checkOptions (options) { if (typeof options.column !== 'string') { throw new Error(`Aggregation expects 'column' in dataview options`); } @@ -294,7 +294,8 @@ module.exports = class Aggregation extends BaseDataview { const self = this; const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`); const value = this.aggregation !== 'count' && this.aggregationColumn ? - this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; + `${this.aggregation}(${this.aggregationColumn})` : + 'count(1)'; // TODO unfiltered will be wrong as filters are already applied at this point const query = searchQueryTpl({ From 51907b95459b653af790f7c137d5edc9bf867906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 16:56:55 +0200 Subject: [PATCH 110/177] Apply extract method to condition --- lib/cartodb/models/dataview/aggregation.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index f621e536..7f6c198a 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -236,7 +236,7 @@ module.exports = class Aggregation extends BaseDataview { override = {}; } - if (this.aggregationColumn && this._isFloatColumn === null) { + if (this._shouldCheckColumnType()) { this._isFloatColumn = false; this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { if (!err && !!type) { @@ -266,6 +266,10 @@ module.exports = class Aggregation extends BaseDataview { return callback(null, aggregationSql); } + _shouldCheckColumnType () { + return this.aggregationColumn && this._isFloatColumn === null; + } + format (result) { const { count = 0, From 9a4794ee10f6aeef8782c837aa0bccd649f158fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:09:55 +0200 Subject: [PATCH 111/177] Remove dot requirement and use template strings to build list aggregation query --- lib/cartodb/models/dataview/list.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index 33993777..b241819e 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -1,11 +1,8 @@ -var dot = require('dot'); -dot.templateSettings.strip = false; - var BaseWidget = require('./base'); var TYPE = 'list'; -var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list'); +var listSqlTpl = ctx => `select ${ctx._columns} from (${ctx._query}) as _cdb_list`; /** { From cdc56e703c11f01ed074e814be446e49684b6112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:10:51 +0200 Subject: [PATCH 112/177] Rename BaseWidget by BaseDataview --- lib/cartodb/models/dataview/list.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index b241819e..064a4cdc 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -1,4 +1,4 @@ -var BaseWidget = require('./base'); +var BaseDataview = require('./base'); var TYPE = 'list'; @@ -20,13 +20,13 @@ function List(query, options) { throw new Error('List expects `columns` array in widget options'); } - BaseWidget.apply(this); + BaseDataview.apply(this); this.query = query; this.columns = options.columns; } -List.prototype = new BaseWidget(); +List.prototype = new BaseDataview(); List.prototype.constructor = List; module.exports = List; From c8ff61c531d02717b5a159d1cb87b58c2cb368e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:15:43 +0200 Subject: [PATCH 113/177] Use ES6 class syntax --- lib/cartodb/models/dataview/list.js | 78 ++++++++++++++--------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index 064a4cdc..cb895429 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -12,52 +12,50 @@ var listSqlTpl = ctx => `select ${ctx._columns} from (${ctx._query}) as _cdb_lis } } */ +module.exports = class List extends BaseDataview{ + constructor (query, options = {}) { + super(); -function List(query, options) { - options = options || {}; + this._checkOptions(options); - if (!Array.isArray(options.columns)) { - throw new Error('List expects `columns` array in widget options'); + this.query = query; + this.columns = options.columns; } - BaseDataview.apply(this); - - this.query = query; - this.columns = options.columns; -} - -List.prototype = new BaseDataview(); -List.prototype.constructor = List; - -module.exports = List; - -List.prototype.sql = function(psql, override, callback) { - if (!callback) { - callback = override; + _checkOptions (options) { + if (!Array.isArray(options.columns)) { + throw new Error('List expects `columns` array in widget options'); + } } - var listSql = listSqlTpl({ - _query: this.query, - _columns: this.columns.join(', ') - }); + sql (psql, override, callback) { + if (!callback) { + callback = override; + } - return callback(null, listSql); -}; + var listSql = listSqlTpl({ + _query: this.query, + _columns: this.columns.join(', ') + }); -List.prototype.format = function(result) { - return { - rows: result.rows + return callback(null, listSql); + } + + format (result) { + return { + rows: result.rows + }; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _columns: this.columns.join(', ') + }); }; -}; - -List.prototype.getType = function() { - return TYPE; -}; - -List.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _columns: this.columns.join(', ') - }); -}; +} From 050e9776d1355e9c24fed37d71fe9ef32b583dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:18:20 +0200 Subject: [PATCH 114/177] Use const for requirements, constants and variables that are initialized once --- lib/cartodb/models/dataview/list.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index cb895429..d3b92e16 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -1,8 +1,8 @@ -var BaseDataview = require('./base'); +const BaseDataview = require('./base'); -var TYPE = 'list'; +const TYPE = 'list'; -var listSqlTpl = ctx => `select ${ctx._columns} from (${ctx._query}) as _cdb_list`; +const listSqlTpl = ctx => `select ${ctx._columns} from (${ctx._query}) as _cdb_list`; /** { @@ -33,7 +33,7 @@ module.exports = class List extends BaseDataview{ callback = override; } - var listSql = listSqlTpl({ + const listSql = listSqlTpl({ _query: this.query, _columns: this.columns.join(', ') }); From d328b534a5622f76489a12177565dd55f4fae0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:19:16 +0200 Subject: [PATCH 115/177] Replace widget word by dataview --- lib/cartodb/models/dataview/list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index d3b92e16..05cee5f7 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -24,7 +24,7 @@ module.exports = class List extends BaseDataview{ _checkOptions (options) { if (!Array.isArray(options.columns)) { - throw new Error('List expects `columns` array in widget options'); + throw new Error('List expects `columns` array in dataview options'); } } From d0e2c9f898e6dff28a1fc92f5d194e916f89d124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:21:21 +0200 Subject: [PATCH 116/177] Use debug module to print sql for debugging purposes --- lib/cartodb/models/dataview/list.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index 05cee5f7..b8d90d1b 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -1,4 +1,5 @@ const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:list'); const TYPE = 'list'; @@ -12,7 +13,7 @@ const listSqlTpl = ctx => `select ${ctx._columns} from (${ctx._query}) as _cdb_l } } */ -module.exports = class List extends BaseDataview{ +module.exports = class List extends BaseDataview { constructor (query, options = {}) { super(); @@ -38,6 +39,8 @@ module.exports = class List extends BaseDataview{ _columns: this.columns.join(', ') }); + debug(listSql); + return callback(null, listSql); } From 2a0b15f085d9c29a954544e15a2fb39dc580d35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:22:34 +0200 Subject: [PATCH 117/177] Remove prefix '_' while passing context param to templates --- lib/cartodb/models/dataview/list.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index b8d90d1b..db90d706 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -3,7 +3,7 @@ const debug = require('debug')('windshaft:dataview:list'); const TYPE = 'list'; -const listSqlTpl = ctx => `select ${ctx._columns} from (${ctx._query}) as _cdb_list`; +const listSqlTpl = ctx => `select ${ctx.columns} from (${ctx.query}) as _cdb_list`; /** { @@ -35,8 +35,8 @@ module.exports = class List extends BaseDataview { } const listSql = listSqlTpl({ - _query: this.query, - _columns: this.columns.join(', ') + query: this.query, + columns: this.columns.join(', ') }); debug(listSql); From dbb23bf9f087147a923f3916d5bd5ecf5d5ba3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:24:13 +0200 Subject: [PATCH 118/177] Remove jshint's complaints --- lib/cartodb/models/dataview/list.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index db90d706..a54c660f 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -60,5 +60,5 @@ module.exports = class List extends BaseDataview { _query: this.query, _columns: this.columns.join(', ') }); - }; -} + } +}; From dcb9b8ec5213b570d53b789fedc60109c247900e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:56:17 +0200 Subject: [PATCH 119/177] Rename BaseWidget by BaseDataview --- lib/cartodb/models/dataview/formula.js | 51 +++++++++++++++----------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 7ec356b7..6781fba3 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -1,26 +1,35 @@ var _ = require('underscore'); -var BaseWidget = require('./base'); +var BaseDataview = require('./base'); var debug = require('debug')('windshaft:widget:formula'); -var dot = require('dot'); -dot.templateSettings.strip = false; +const countInfinitiesQueryTpl = ctx => ` + SELECT count(1) FROM (${ctx._query}) __cdb_formula_infinities + WHERE ${ctx._column} = 'infinity'::float OR ${ctx._column} = '-infinity'::float +`; -var formulaQueryTpl = dot.template([ - 'SELECT', - ' {{=it._operation}}({{=it._column}}) AS result,', - ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' -].join('\n')); +const countNansQueryTpl = ctx => ` + SELECT count(1) FROM (${ctx._query}) __cdb_formula_nans + WHERE ${ctx._column} = 'NaN'::float +`; + +const filterOutSpecialNumericValuesTpl = ctx => ` + WHERE + ${ctx._column} != 'infinity'::float + AND + ${ctx._column} != '-infinity'::float + AND + ${ctx._column} != 'NaN'::float +`; + +const formulaQueryTpl = ctx => ` + SELECT + ${ctx._operation}(${ctx._column}) AS result, + (SELECT count(1) FROM (${ctx._query}) _cdb_formula_nulls WHERE ${ctx._column} IS NULL) AS nulls_count + ${ctx._isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''} + ${ctx._isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''} + FROM (${ctx._query}) __cdb_formula + ${ctx._isFloatColumn && ctx._operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''} +`; var VALID_OPERATIONS = { count: true, @@ -54,7 +63,7 @@ function Formula(query, options, queries) { throw new Error('Formula expects `column` in widget options'); } - BaseWidget.apply(this); + BaseDataview.apply(this); this.query = query; this.queries = queries; @@ -63,7 +72,7 @@ function Formula(query, options, queries) { this._isFloatColumn = null; } -Formula.prototype = new BaseWidget(); +Formula.prototype = new BaseDataview(); Formula.prototype.constructor = Formula; module.exports = Formula; From 1063d81c1bb64a71c1ccc14f0626005b7887f3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:56:40 +0200 Subject: [PATCH 120/177] Rename debug namespace --- lib/cartodb/models/dataview/formula.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 6781fba3..54bbe430 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -1,6 +1,6 @@ var _ = require('underscore'); var BaseDataview = require('./base'); -var debug = require('debug')('windshaft:widget:formula'); +var debug = require('debug')('windshaft:dataview:formula'); const countInfinitiesQueryTpl = ctx => ` SELECT count(1) FROM (${ctx._query}) __cdb_formula_infinities From a4ecc18f2fb98db46b3b8ab400f1fc5cee4836d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 17:57:24 +0200 Subject: [PATCH 121/177] Use default values for constructor's arguments --- lib/cartodb/models/dataview/formula.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 54bbe430..4c7f185b 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -50,7 +50,7 @@ var TYPE = 'formula'; } } */ -function Formula(query, options, queries) { +function Formula(query, options = {}, queries = {}) { if (!_.isString(options.operation)) { throw new Error('Formula expects `operation` in widget options'); } From 9c64d674b3522f80c812f7676fb1c1aa4c3c5dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 14 Sep 2017 18:02:13 +0200 Subject: [PATCH 122/177] Do not use underscore --- lib/cartodb/models/dataview/formula.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 4c7f185b..36b5369b 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -1,4 +1,3 @@ -var _ = require('underscore'); var BaseDataview = require('./base'); var debug = require('debug')('windshaft:dataview:formula'); @@ -51,7 +50,7 @@ var TYPE = 'formula'; } */ function Formula(query, options = {}, queries = {}) { - if (!_.isString(options.operation)) { + if (typeof options.operation !== 'string') { throw new Error('Formula expects `operation` in widget options'); } @@ -59,7 +58,7 @@ function Formula(query, options = {}, queries = {}) { throw new Error("Formula does not support '" + options.operation + "' operation"); } - if (options.operation !== 'count' && !_.isString(options.column)) { + if (options.operation !== 'count' && typeof options.column !== 'string') { throw new Error('Formula expects `column` in widget options'); } From 2437288d9d1675f226a6c8e0aa6f7c4dc233d7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 10:37:51 +0200 Subject: [PATCH 123/177] Replace widget word by dataview --- lib/cartodb/models/dataview/formula.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 36b5369b..b96e05b1 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -51,15 +51,15 @@ var TYPE = 'formula'; */ function Formula(query, options = {}, queries = {}) { if (typeof options.operation !== 'string') { - throw new Error('Formula expects `operation` in widget options'); + throw new Error(`Formula expects 'operation' in dataview options`); } if (!VALID_OPERATIONS[options.operation]) { - throw new Error("Formula does not support '" + options.operation + "' operation"); + throw new Error(`Formula does not support '${options.operation}' operation`) } if (options.operation !== 'count' && typeof options.column !== 'string') { - throw new Error('Formula expects `column` in widget options'); + throw new Error(`Formula expects 'column' in dataview options`); } BaseDataview.apply(this); From 68dfed8b85df674685c7e3569f2d2697e2f88bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 10:48:44 +0200 Subject: [PATCH 124/177] Use ES6 class syntax --- lib/cartodb/models/dataview/formula.js | 146 ++++++++++++------------- 1 file changed, 71 insertions(+), 75 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index b96e05b1..ba6725e3 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -49,93 +49,89 @@ var TYPE = 'formula'; } } */ -function Formula(query, options = {}, queries = {}) { - if (typeof options.operation !== 'string') { - throw new Error(`Formula expects 'operation' in dataview options`); +module.exports = class Formula extends BaseDataview { + constructor (query, options = {}, queries = {}) { + if (typeof options.operation !== 'string') { + throw new Error(`Formula expects 'operation' in dataview options`); + } + + if (!VALID_OPERATIONS[options.operation]) { + throw new Error(`Formula does not support '${options.operation}' operation`) + } + + if (options.operation !== 'count' && typeof options.column !== 'string') { + throw new Error(`Formula expects 'column' in dataview options`); + } + + this.query = query; + this.queries = queries; + this.column = options.column || '1'; + this.operation = options.operation; + this._isFloatColumn = null; } - if (!VALID_OPERATIONS[options.operation]) { - throw new Error(`Formula does not support '${options.operation}' operation`) - } - if (options.operation !== 'count' && typeof options.column !== 'string') { - throw new Error(`Formula expects 'column' in dataview options`); - } + sql (psql, override, callback) { + var self = this; - BaseDataview.apply(this); + if (!callback) { + callback = override; + override = {}; + } - this.query = query; - this.queries = queries; - this.column = options.column || '1'; - this.operation = options.operation; - this._isFloatColumn = null; -} + if (this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } -Formula.prototype = new BaseDataview(); -Formula.prototype.constructor = Formula; - -module.exports = Formula; - -Formula.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; - } - - if (this._isFloatColumn === null) { - this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { - if (!err && !!type) { - self._isFloatColumn = type.float; - } - self.sql(psql, override, callback); + var formulaSql = formulaQueryTpl({ + _isFloatColumn: this._isFloatColumn, + _query: this.query, + _operation: this.operation, + _column: this.column }); - return null; + + debug(formulaSql); + + return callback(null, formulaSql); } - var formulaSql = formulaQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.query, - _operation: this.operation, - _column: this.column - }); + format (result) { + var formattedResult = { + operation: this.operation, + result: 0, + nulls: 0, + nans: 0, + infinities: 0 + }; - debug(formulaSql); + if (result.rows.length) { + formattedResult.operation = this.operation; + formattedResult.result = result.rows[0].result; + formattedResult.nulls = result.rows[0].nulls_count; + formattedResult.nans = result.rows[0].nans_count; + formattedResult.infinities = result.rows[0].infinities_count; + } - return callback(null, formulaSql); -}; + return formattedResult; + } -Formula.prototype.format = function(result) { - var formattedResult = { - operation: this.operation, - result: 0, - nulls: 0, - nans: 0, - infinities: 0 + getType () { + return TYPE; }; - if (result.rows.length) { - formattedResult.operation = this.operation; - formattedResult.result = result.rows[0].result; - formattedResult.nulls = result.rows[0].nulls_count; - formattedResult.nans = result.rows[0].nans_count; - formattedResult.infinities = result.rows[0].infinities_count; + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _operation: this.operation + }); } - - return formattedResult; -}; - -Formula.prototype.getType = function() { - return TYPE; -}; - -Formula.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _column: this.column, - _operation: this.operation - }); -}; +} From ecbc7a28e769bb273556b56d8730121b5c8c185b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 10:49:20 +0200 Subject: [PATCH 125/177] Declare constants with const keyword --- lib/cartodb/models/dataview/formula.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index ba6725e3..efaeb0c6 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -30,7 +30,7 @@ const formulaQueryTpl = ctx => ` ${ctx._isFloatColumn && ctx._operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''} `; -var VALID_OPERATIONS = { +const VALID_OPERATIONS = { count: true, avg: true, sum: true, @@ -38,7 +38,7 @@ var VALID_OPERATIONS = { max: true }; -var TYPE = 'formula'; +const TYPE = 'formula'; /** { From c00a93f4144abf7eccaf9c7691c2138218866950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 10:58:11 +0200 Subject: [PATCH 126/177] Use destruturing assignment to format the formula result --- lib/cartodb/models/dataview/formula.js | 27 ++++++++++++-------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index efaeb0c6..2172beab 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -103,23 +103,20 @@ module.exports = class Formula extends BaseDataview { } format (result) { - var formattedResult = { + const { + result = 0, + nulls_count = 0, + nans_count = 0, + infinities_count = 0 + } = result.rows[0] || {}; + + return { operation: this.operation, - result: 0, - nulls: 0, - nans: 0, - infinities: 0 + result, + nulls: nulls_count, + nans: nans_count, + infinities: infinities_count }; - - if (result.rows.length) { - formattedResult.operation = this.operation; - formattedResult.result = result.rows[0].result; - formattedResult.nulls = result.rows[0].nulls_count; - formattedResult.nans = result.rows[0].nans_count; - formattedResult.infinities = result.rows[0].infinities_count; - } - - return formattedResult; } getType () { From 9771979b8fc391dc396f3c45be1effb6659503d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 10:59:07 +0200 Subject: [PATCH 127/177] Missing call to super class in constructor --- lib/cartodb/models/dataview/formula.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 2172beab..c2528a91 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -51,6 +51,8 @@ const TYPE = 'formula'; */ module.exports = class Formula extends BaseDataview { constructor (query, options = {}, queries = {}) { + super(); + if (typeof options.operation !== 'string') { throw new Error(`Formula expects 'operation' in dataview options`); } From 11f7b38c6975144ff3ac9abfb0aae19f085e23c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 11:54:56 +0200 Subject: [PATCH 128/177] Do not use dot module to build column type query --- lib/cartodb/models/dataview/base.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 29069d37..fb5f520e 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -1,6 +1,3 @@ -var dot = require('dot'); -dot.templateSettings.strip = false; - function BaseDataview() {} module.exports = BaseDataview; @@ -43,15 +40,14 @@ var DATE_OIDS = { 1184: true }; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1' -); +var columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`; BaseDataview.prototype.getColumnType = function (psql, column, query, callback) { var readOnlyTransaction = true; var columnTypeQuery = columnTypeQueryTpl({ - column: column, query: query + column: column, + query: query }); psql.query(columnTypeQuery, function(err, result) { From b4ce13e4293c5770d4852415c2e1e33f1f7d4219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 11:56:59 +0200 Subject: [PATCH 129/177] Use object shorthand notation --- lib/cartodb/models/dataview/base.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index fb5f520e..cacfaa77 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -45,10 +45,7 @@ var columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ct BaseDataview.prototype.getColumnType = function (psql, column, query, callback) { var readOnlyTransaction = true; - var columnTypeQuery = columnTypeQueryTpl({ - column: column, - query: query - }); + var columnTypeQuery = columnTypeQueryTpl({ column, query }); psql.query(columnTypeQuery, function(err, result) { if (err) { From ada58f6ea2a5c8590193ab67359655f78af69f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 13:35:00 +0200 Subject: [PATCH 130/177] Use const keyword to declare varibles --- lib/cartodb/models/dataview/base.js | 91 ++++++++++++++--------------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index cacfaa77..bc7d13bd 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -1,60 +1,16 @@ -function BaseDataview() {} - -module.exports = BaseDataview; - -BaseDataview.prototype.getResult = function(psql, override, callback) { - var self = this; - this.sql(psql, override, function(err, query) { - if (err) { - return callback(err); - } - - psql.query(query, function(err, result) { - if (err) { - return callback(err, result); - } - - result = self.format(result, override); - result.type = self.getType(); - - return callback(null, result); - - }, true); // use read-only transaction - }); - -}; - -BaseDataview.prototype.search = function(psql, userQuery, callback) { - return callback(null, this.format({ rows: [] })); -}; - -var FLOAT_OIDS = { +const FLOAT_OIDS = { 700: true, 701: true, 1700: true }; -var DATE_OIDS = { +const DATE_OIDS = { 1082: true, 1114: true, 1184: true }; -var columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`; - -BaseDataview.prototype.getColumnType = function (psql, column, query, callback) { - var readOnlyTransaction = true; - - var columnTypeQuery = columnTypeQueryTpl({ column, query }); - - psql.query(columnTypeQuery, function(err, result) { - if (err) { - return callback(err); - } - var pgType = result.rows[0].pg_typeof; - callback(null, getPGTypeName(pgType)); - }, readOnlyTransaction); -}; +const columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`; function getPGTypeName (pgType) { return { @@ -62,3 +18,44 @@ function getPGTypeName (pgType) { date: DATE_OIDS.hasOwnProperty(pgType) }; } + +module.exports = class BaseDataview { + getResult (psql, override, callback) { + const self = this; + this.sql(psql, override, function(err, query) { + if (err) { + return callback(err); + } + + psql.query(query, function(err, result) { + if (err) { + return callback(err, result); + } + + result = self.format(result, override); + result.type = self.getType(); + + return callback(null, result); + + }, true); // use read-only transaction + }); + } + + search (psql, userQuery, callback) { + return callback(null, this.format({ rows: [] })); + }; + + getColumnType (psql, column, query, callback) { + const readOnlyTransaction = true; + + const columnTypeQuery = columnTypeQueryTpl({ column, query }); + + psql.query(columnTypeQuery, function(err, result) { + if (err) { + return callback(err); + } + const pgType = result.rows[0].pg_typeof; + callback(null, getPGTypeName(pgType)); + }, readOnlyTransaction); + } +}; From 90c4796d4ede54f649f0f8d7112afae325c32b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 13:41:54 +0200 Subject: [PATCH 131/177] Remove empty line --- lib/cartodb/models/dataview/base.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index bc7d13bd..702f43e5 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -43,11 +43,10 @@ module.exports = class BaseDataview { search (psql, userQuery, callback) { return callback(null, this.format({ rows: [] })); - }; + } getColumnType (psql, column, query, callback) { const readOnlyTransaction = true; - const columnTypeQuery = columnTypeQueryTpl({ column, query }); psql.query(columnTypeQuery, function(err, result) { From 5d632d936eab70d28e69b7e3226a4ecd62df8eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 14:04:09 +0200 Subject: [PATCH 132/177] Use ES6 class syntax --- lib/cartodb/models/dataview/factory.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/factory.js b/lib/cartodb/models/dataview/factory.js index 50814f23..70063107 100644 --- a/lib/cartodb/models/dataview/factory.js +++ b/lib/cartodb/models/dataview/factory.js @@ -1,12 +1,14 @@ var dataviews = require('./'); -var DataviewFactory = { - dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) { - allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName]; - return allDataviews; - }, {}), +module.exports = class DataviewFactory { + static get dataviews() { + return Object.keys(dataviews).reduce((allDataviews, dataviewClassName) => { + allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName]; + return allDataviews; + }, {}); + } - getDataview: function(query, dataviewDefinition) { + static getDataview (query, dataviewDefinition) { var type = dataviewDefinition.type; if (!this.dataviews[type]) { throw new Error('Invalid dataview type: "' + type + '"'); @@ -14,5 +16,3 @@ var DataviewFactory = { return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql); } }; - -module.exports = DataviewFactory; From d5d9044686a1ee5eaa01ced5e013029f5b61d63f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 14:05:23 +0200 Subject: [PATCH 133/177] Use const keyword to declare variables --- lib/cartodb/models/dataview/factory.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/factory.js b/lib/cartodb/models/dataview/factory.js index 70063107..6ad65312 100644 --- a/lib/cartodb/models/dataview/factory.js +++ b/lib/cartodb/models/dataview/factory.js @@ -1,4 +1,4 @@ -var dataviews = require('./'); +const dataviews = require('./'); module.exports = class DataviewFactory { static get dataviews() { @@ -9,7 +9,7 @@ module.exports = class DataviewFactory { } static getDataview (query, dataviewDefinition) { - var type = dataviewDefinition.type; + const type = dataviewDefinition.type; if (!this.dataviews[type]) { throw new Error('Invalid dataview type: "' + type + '"'); } From ef5049f28f5a57e9610e2c8a50e99b66280de61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 14:07:46 +0200 Subject: [PATCH 134/177] Use destructuring assignment to improve readability --- lib/cartodb/models/dataview/factory.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/factory.js b/lib/cartodb/models/dataview/factory.js index 6ad65312..fa55641c 100644 --- a/lib/cartodb/models/dataview/factory.js +++ b/lib/cartodb/models/dataview/factory.js @@ -9,10 +9,12 @@ module.exports = class DataviewFactory { } static getDataview (query, dataviewDefinition) { - const type = dataviewDefinition.type; + const { type, options, sql } = dataviewDefinition; + if (!this.dataviews[type]) { throw new Error('Invalid dataview type: "' + type + '"'); } - return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql); + + return new this.dataviews[type](query, options, sql); } }; From 1959a841fd8823383f5e37e3bf7028a2f71377b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 14:26:22 +0200 Subject: [PATCH 135/177] Use arrow function to take advantage of bound context --- lib/cartodb/models/dataview/base.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 702f43e5..ea7f9474 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -21,8 +21,7 @@ function getPGTypeName (pgType) { module.exports = class BaseDataview { getResult (psql, override, callback) { - const self = this; - this.sql(psql, override, function(err, query) { + this.sql(psql, override, (err, query) => { if (err) { return callback(err); } @@ -32,8 +31,8 @@ module.exports = class BaseDataview { return callback(err, result); } - result = self.format(result, override); - result.type = self.getType(); + result = this.format(result, override); + result.type = this.getType(); return callback(null, result); From c7ed3d34e899ba6b76ee96755d85b140ab0c5725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 11:35:06 +0200 Subject: [PATCH 136/177] Use const instead of var to declare variables --- lib/cartodb/models/dataview/formula.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index c2528a91..1ce0050b 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -1,5 +1,5 @@ -var BaseDataview = require('./base'); -var debug = require('debug')('windshaft:dataview:formula'); +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:formula'); const countInfinitiesQueryTpl = ctx => ` SELECT count(1) FROM (${ctx._query}) __cdb_formula_infinities @@ -58,7 +58,7 @@ module.exports = class Formula extends BaseDataview { } if (!VALID_OPERATIONS[options.operation]) { - throw new Error(`Formula does not support '${options.operation}' operation`) + throw new Error(`Formula does not support '${options.operation}' operation`); } if (options.operation !== 'count' && typeof options.column !== 'string') { @@ -74,7 +74,7 @@ module.exports = class Formula extends BaseDataview { sql (psql, override, callback) { - var self = this; + const self = this; if (!callback) { callback = override; @@ -92,7 +92,7 @@ module.exports = class Formula extends BaseDataview { return null; } - var formulaSql = formulaQueryTpl({ + const formulaSql = formulaQueryTpl({ _isFloatColumn: this._isFloatColumn, _query: this.query, _operation: this.operation, @@ -123,7 +123,7 @@ module.exports = class Formula extends BaseDataview { getType () { return TYPE; - }; + } toString () { return JSON.stringify({ @@ -133,4 +133,4 @@ module.exports = class Formula extends BaseDataview { _operation: this.operation }); } -} +}; From 419b29e60900cf5e6100835822899f7cc5fa3d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 11:39:06 +0200 Subject: [PATCH 137/177] Do not prefix with '_' template context --- lib/cartodb/models/dataview/formula.js | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 1ce0050b..b6faeb55 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -2,32 +2,32 @@ const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:formula'); const countInfinitiesQueryTpl = ctx => ` - SELECT count(1) FROM (${ctx._query}) __cdb_formula_infinities - WHERE ${ctx._column} = 'infinity'::float OR ${ctx._column} = '-infinity'::float + SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities + WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float `; const countNansQueryTpl = ctx => ` - SELECT count(1) FROM (${ctx._query}) __cdb_formula_nans - WHERE ${ctx._column} = 'NaN'::float + SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans + WHERE ${ctx.column} = 'NaN'::float `; const filterOutSpecialNumericValuesTpl = ctx => ` WHERE - ${ctx._column} != 'infinity'::float + ${ctx.column} != 'infinity'::float AND - ${ctx._column} != '-infinity'::float + ${ctx.column} != '-infinity'::float AND - ${ctx._column} != 'NaN'::float + ${ctx.column} != 'NaN'::float `; const formulaQueryTpl = ctx => ` SELECT - ${ctx._operation}(${ctx._column}) AS result, - (SELECT count(1) FROM (${ctx._query}) _cdb_formula_nulls WHERE ${ctx._column} IS NULL) AS nulls_count - ${ctx._isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''} - ${ctx._isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''} - FROM (${ctx._query}) __cdb_formula - ${ctx._isFloatColumn && ctx._operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''} + ${ctx.operation}(${ctx.column}) AS result, + (SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count + ${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''} + ${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''} + FROM (${ctx.query}) __cdb_formula + ${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''} `; const VALID_OPERATIONS = { @@ -93,10 +93,10 @@ module.exports = class Formula extends BaseDataview { } const formulaSql = formulaQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.query, - _operation: this.operation, - _column: this.column + isFloatColumn: this._isFloatColumn, + query: this.query, + operation: this.operation, + column: this.column }); debug(formulaSql); From d4bb4edd1d27fba4c2ab533035c70c2762cbaf1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 11:44:20 +0200 Subject: [PATCH 138/177] Applyy extract method to check input options --- lib/cartodb/models/dataview/formula.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index b6faeb55..b167b87e 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -53,6 +53,16 @@ module.exports = class Formula extends BaseDataview { constructor (query, options = {}, queries = {}) { super(); + this._checkOptions(options); + + this.query = query; + this.queries = queries; + this.column = options.column || '1'; + this.operation = options.operation; + this._isFloatColumn = null; + } + + _checkOptions (options) { if (typeof options.operation !== 'string') { throw new Error(`Formula expects 'operation' in dataview options`); } @@ -64,12 +74,6 @@ module.exports = class Formula extends BaseDataview { if (options.operation !== 'count' && typeof options.column !== 'string') { throw new Error(`Formula expects 'column' in dataview options`); } - - this.query = query; - this.queries = queries; - this.column = options.column || '1'; - this.operation = options.operation; - this._isFloatColumn = null; } From 64c3e68303c76bc913917b6fba672cdf2a41200a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 14:48:54 +0200 Subject: [PATCH 139/177] Fix double declaration of 'result' --- lib/cartodb/models/dataview/formula.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index b167b87e..1ad08684 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -108,13 +108,13 @@ module.exports = class Formula extends BaseDataview { return callback(null, formulaSql); } - format (result) { + format (res) { const { result = 0, nulls_count = 0, nans_count = 0, infinities_count = 0 - } = result.rows[0] || {}; + } = res.rows[0] || {}; return { operation: this.operation, From 61ea05d1c2b73cf0d0663bc4d4648cc83f13c5f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 15 Sep 2017 14:51:02 +0200 Subject: [PATCH 140/177] Do not assign a value by default for special float values counters --- lib/cartodb/models/dataview/formula.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 1ad08684..c0829fa5 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -112,8 +112,8 @@ module.exports = class Formula extends BaseDataview { const { result = 0, nulls_count = 0, - nans_count = 0, - infinities_count = 0 + nans_count, + infinities_count } = res.rows[0] || {}; return { From f63fab40ed5ae6715f3f5b85ebc24b2bcd8b1ce5 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 18 Sep 2017 11:34:18 +0200 Subject: [PATCH 141/177] Removing 'self' vars using arrow functions --- lib/cartodb/models/dataview/aggregation.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 7f6c198a..b4c59af2 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -229,8 +229,6 @@ module.exports = class Aggregation extends BaseDataview { } sql (psql, override, callback) { - const self = this; - if (!callback) { callback = override; override = {}; @@ -238,11 +236,11 @@ module.exports = class Aggregation extends BaseDataview { if (this._shouldCheckColumnType()) { this._isFloatColumn = false; - this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => { if (!err && !!type) { - self._isFloatColumn = type.float; + this._isFloatColumn = type.float; } - self.sql(psql, override, callback); + this.sql(psql, override, callback); }); return null; } @@ -295,7 +293,6 @@ module.exports = class Aggregation extends BaseDataview { } search (psql, userQuery, callback) { - const self = this; const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`); const value = this.aggregation !== 'count' && this.aggregationColumn ? `${this.aggregation}(${this.aggregationColumn})` : @@ -319,12 +316,12 @@ module.exports = class Aggregation extends BaseDataview { debug(query); - psql.query(query, function(err, result) { + psql.query(query, (err, result) => { if (err) { return callback(err, result); } - return callback(null, {type: self.getType(), categories: result.rows }); + return callback(null, {type: this.getType(), categories: result.rows }); }, true); // use read-only transaction } From 222cfb90fdbc949160656b7a75dde1e44f5250c7 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 18 Sep 2017 12:20:59 +0200 Subject: [PATCH 142/177] Removing 'self' vars using arrow functions --- lib/cartodb/models/dataview/formula.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index c0829fa5..daaeaf0c 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -78,8 +78,6 @@ module.exports = class Formula extends BaseDataview { sql (psql, override, callback) { - const self = this; - if (!callback) { callback = override; override = {}; @@ -87,11 +85,11 @@ module.exports = class Formula extends BaseDataview { if (this._isFloatColumn === null) { this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { if (!err && !!type) { - self._isFloatColumn = type.float; + this._isFloatColumn = type.float; } - self.sql(psql, override, callback); + this.sql(psql, override, callback); }); return null; } From c734f43643e75f5cae15fa84ad658e497eb4326c Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 25 Sep 2017 11:11:05 +0000 Subject: [PATCH 143/177] Upgrades camshaft, cartodb-query-tables, and turbo-carto Better support for query variables. --- NEWS.md | 4 +++- package.json | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 2efa11b6..2971a93e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,9 @@ # Changelog -## 3.12.11 +## 3.13.0 Released 2017-mm-dd + - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. + ## 3.12.10 Released 2017-09-18 diff --git a/package.json b/package.json index 9c70ad72..ea4f49e7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.11", + "version": "3.13.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -23,9 +23,9 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.58.1", + "camshaft": "0.59.0", "cartodb-psql": "0.10.1", - "cartodb-query-tables": "0.2.0", + "cartodb-query-tables": "0.3.0", "cartodb-redis": "0.14.0", "debug": "~2.2.0", "dot": "~1.0.2", @@ -40,7 +40,7 @@ "request": "~2.79.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.19.2", + "turbo-carto": "0.20.0", "underscore": "~1.6.0", "windshaft": "3.3.2", "yargs": "~5.0.0" From d3dafc8a408ba89dad1931114005b8cb246a0ba8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 25 Sep 2017 11:24:33 +0000 Subject: [PATCH 144/177] Regenerate lock file --- yarn.lock | 315 +++++++++++++++++++++++++++--------------------------- 1 file changed, 157 insertions(+), 158 deletions(-) diff --git a/yarn.lock b/yarn.lock index a9ba816b..8fbdeeb4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -32,6 +32,15 @@ ajv@^4.9.1: co "^4.6.0" json-stable-stringify "^1.0.1" +ajv@^5.1.0: + version "5.2.2" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.2.tgz#47c68d69e86f5d953103b0074a9430dc63da5e39" + dependencies: + co "^4.6.0" + fast-deep-equal "^1.0.0" + json-schema-traverse "^0.3.0" + json-stable-stringify "^1.0.1" + align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -53,8 +62,8 @@ ansi-styles@^2.2.1: resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" aproba@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" + version "1.2.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" are-we-there-yet@~1.1.2: version "1.1.4" @@ -105,7 +114,11 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws4@^1.2.1: +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + +aws4@^1.2.1, aws4@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" @@ -150,6 +163,18 @@ boom@2.x.x: dependencies: hoek "2.x.x" +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" + dependencies: + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + dependencies: + hoek "4.x.x" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -194,9 +219,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.58.1: - version "0.58.1" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.1.tgz#e4156580683f624212ea3020e59790ad006f24cc" +camshaft@0.59.0: + version "0.59.0" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.0.tgz#699c0834116ec87e067d42e11eea9540bddc6383" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -253,9 +278,9 @@ cartodb-psql@0.10.1, cartodb-psql@^0.10.1: pg cartodb/node-postgres#6.1.6-cdb1 underscore "~1.6.0" -cartodb-query-tables@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" +cartodb-query-tables@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756" cartodb-redis@0.14.0: version "0.14.0" @@ -393,6 +418,12 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" +cryptiles@3.x.x: + version "3.1.2" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" + dependencies: + boom "5.x.x" + d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -426,8 +457,8 @@ debug@^1.0.4: ms "2.0.0" debug@^2.2.0: - version "2.6.8" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: ms "2.0.0" @@ -441,7 +472,7 @@ deep-eql@^0.1.3: dependencies: type-detect "0.1.1" -deep-equal@^1.0.0, deep-equal@~1.0.1: +deep-equal@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" @@ -453,17 +484,6 @@ deep-is@~0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" -define-properties@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94" - dependencies: - foreach "^2.0.5" - object-keys "^1.0.8" - -defined@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" - delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -550,24 +570,6 @@ error-ex@^1.2.0: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.5.0: - version "1.8.2" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.8.2.tgz#25103263dc4decbda60e0c737ca32313518027ee" - dependencies: - es-to-primitive "^1.1.1" - function-bind "^1.1.1" - has "^1.0.1" - is-callable "^1.1.3" - is-regex "^1.0.4" - -es-to-primitive@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.1.1.tgz#45355248a88979034b6792e19bb81f2b7975dd0d" - dependencies: - is-callable "^1.1.1" - is-date-object "^1.0.1" - is-symbol "^1.0.1" - es6-promise@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.1.2.tgz#795e25ceb47f7babb263d151afbedd92d18e6a07" @@ -645,7 +647,7 @@ express@~4.13.3: utils-merge "1.0.0" vary "~1.0.1" -extend@~3.0.0: +extend@~3.0.0, extend@~3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -653,6 +655,10 @@ extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" +fast-deep-equal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" + fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -679,16 +685,6 @@ find-up@^1.0.0: path-exists "^2.0.0" pinkie-promise "^2.0.0" -for-each@~0.3.2: - version "0.3.2" - resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.2.tgz#2c40450b9348e97f281322593ba96704b9abd4d4" - dependencies: - is-function "~1.0.0" - -foreach@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" - forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -701,6 +697,14 @@ form-data@~2.1.1: combined-stream "^1.0.5" mime-types "^2.1.12" +form-data@~2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.5" + mime-types "^2.1.12" + forwarded@~0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" @@ -730,10 +734,6 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: mkdirp ">=0.5 0" rimraf "2" -function-bind@^1.0.2, function-bind@^1.1.1, function-bind@~1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - gauge@~2.7.3: version "2.7.4" resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" @@ -821,7 +821,7 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1, glob@~7.1.2: +glob@^7.0.5, glob@^7.1.1: version "7.1.2" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: @@ -872,6 +872,10 @@ har-schema@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + har-validator@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d" @@ -888,6 +892,13 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" +har-validator@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" + dependencies: + ajv "^5.1.0" + har-schema "^2.0.0" + has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -902,13 +913,7 @@ has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" -has@^1.0.1, has@~1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.1.tgz#8461733f538b0837c9361e39a9ab9e9704dc2f28" - dependencies: - function-bind "^1.0.2" - -hawk@~3.1.3: +hawk@3.1.3, hawk@~3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" dependencies: @@ -917,6 +922,15 @@ hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" +hawk@~6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -928,6 +942,10 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" +hoek@4.x.x: + version "4.2.0" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" + hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" @@ -957,6 +975,14 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -1002,24 +1028,12 @@ is-builtin-module@^1.0.0: dependencies: builtin-modules "^1.0.0" -is-callable@^1.1.1, is-callable@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.3.tgz#86eb75392805ddc33af71c92a0eedf74ee7604b2" - -is-date-object@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.1.tgz#9aa20eb6aeebbff77fbd33e74ca01b33581d3a16" - is-fullwidth-code-point@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" dependencies: number-is-nan "^1.0.0" -is-function@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-function/-/is-function-1.0.1.tgz#12cfb98b65b57dd3d193a3121f5f6e2f437602b5" - is-my-json-valid@^2.12.4: version "2.16.1" resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11" @@ -1033,16 +1047,6 @@ is-property@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84" -is-regex@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491" - dependencies: - has "^1.0.1" - -is-symbol@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.1.tgz#3cc59f00025194b6ab2e38dbae6689256b660572" - is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -1118,6 +1122,10 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" +json-schema-traverse@^0.3.0: + version "0.3.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" + json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -1331,7 +1339,7 @@ mime-db@~1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, mime-types@~2.1.7: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: @@ -1359,7 +1367,7 @@ minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" -minimist@^1.2.0, minimist@~1.2.0: +minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" @@ -1448,18 +1456,18 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.37: - version "0.6.37" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.37.tgz#3c872b236b2e266e4140578fe1ee88f693323a05" +node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: + version "0.6.38" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.38.tgz#e92a20f83416415bb4086f6d1fb78b3da73d113d" dependencies: + hawk "3.1.3" mkdirp "^0.5.1" nopt "^4.0.1" npmlog "^4.0.2" rc "^1.1.7" - request "^2.81.0" + request "2.81.0" rimraf "^2.6.1" semver "^5.3.0" - tape "^4.6.3" tar "^2.2.1" tar-pack "^3.4.0" @@ -1502,7 +1510,7 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1: +oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" @@ -1514,14 +1522,6 @@ object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" -object-inspect@~1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.3.0.tgz#5b1eb8e6742e2ee83342a637034d844928ba2f6d" - -object-keys@^1.0.8: - version "1.0.11" - resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1601,10 +1601,6 @@ path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" -path-parse@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" - path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -1621,6 +1617,10 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -1776,6 +1776,10 @@ qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" +qs@~6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + queue-async@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" @@ -1872,7 +1876,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@^2.69.0, request@^2.81.0: +request@2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1899,6 +1903,33 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@2.x, request@^2.55.0, request@^2.69.0: + version "2.82.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.82.0.tgz#2ba8a92cd7ac45660ea2b10a53ae67cd247516ea" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + hawk "~6.0.2" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + stringstream "~0.0.5" + tough-cookie "~2.3.2" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" @@ -1936,18 +1967,6 @@ resolve@1.1.x: version "1.1.7" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" -resolve@~1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.4.0.tgz#a75be01c53da25d934a98ebd0e4c4a7312f92a86" - dependencies: - path-parse "^1.0.5" - -resumer@~0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/resumer/-/resumer-0.0.0.tgz#f1e8f461e4064ba39e82af3cdc2a8c893d076759" - dependencies: - through "~2.3.4" - right-align@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/right-align/-/right-align-0.1.3.tgz#61339b722fe6a3515689210d24e14c96148613ef" @@ -1966,7 +1985,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" @@ -2058,6 +2077,12 @@ sntp@1.x.x: dependencies: hoek "2.x.x" +sntp@2.x.x: + version "2.0.2" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" + dependencies: + hoek "4.x.x" + source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -2111,11 +2136,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.11" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.11.tgz#84421b181b3e9f4939b7a3d712263d7946cfc96d" + version "3.1.12" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.12.tgz#2b3a14b17162e39e8aa6e1e2487a41d0795396d8" dependencies: nan "~2.7.0" - node-pre-gyp "~0.6.37" + node-pre-gyp "~0.6.38" srs@1.x: version "1.2.0" @@ -2167,14 +2192,6 @@ string-width@^1.0.1, string-width@^1.0.2: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" -string.prototype.trim@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz#d04de2c89e137f4d7d206f086b5ed2fae6be8cea" - dependencies: - define-properties "^1.1.2" - es-abstract "^1.5.0" - function-bind "^1.0.2" - string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" @@ -2185,7 +2202,7 @@ string_decoder@~1.0.3: dependencies: safe-buffer "~5.1.0" -stringstream@~0.0.4: +stringstream@~0.0.4, stringstream@~0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2225,24 +2242,6 @@ supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: dependencies: has-flag "^1.0.0" -tape@^4.6.3: - version "4.8.0" - resolved "https://registry.yarnpkg.com/tape/-/tape-4.8.0.tgz#f6a9fec41cc50a1de50fa33603ab580991f6068e" - dependencies: - deep-equal "~1.0.1" - defined "~1.0.0" - for-each "~0.3.2" - function-bind "~1.1.0" - glob "~7.1.2" - has "~1.0.1" - inherits "~2.0.3" - minimist "~1.2.0" - object-inspect "~1.3.0" - resolve "~1.4.0" - resumer "~0.0.0" - string.prototype.trim "~1.1.2" - through "~2.3.8" - tar-pack@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.0.tgz#23be2d7f671a8339376cbdb0b8fe3fdebf317984" @@ -2271,7 +2270,7 @@ through2@~0.2.3: readable-stream "~1.1.9" xtend "~2.1.1" -through@2, through@~2.3.4, through@~2.3.8: +through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" @@ -2308,9 +2307,9 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0: - version "2.3.2" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.2.tgz#f081f76e4c85720e6c37a5faced737150d84072a" +tough-cookie@~2.3.0, tough-cookie@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: punycode "^1.4.1" @@ -2324,9 +2323,9 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@0.19.2: - version "0.19.2" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd" +turbo-carto@0.20.0: + version "0.20.0" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.0.tgz#ebeb4620f3e26ed3cce53a9ff30f087f46a2e3fd" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" @@ -2401,7 +2400,7 @@ utils-merge@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" -uuid@^3.0.0: +uuid@^3.0.0, uuid@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" From c9f0902703d76e633f580ef349037d0d2e879214 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 27 Sep 2017 18:44:06 +0200 Subject: [PATCH 145/177] Upgrade camshsft to 0.59.1 This fixes duplicate column names in aggregate-intersection analysis --- NEWS.md | 3 +-- package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8208c0f5..c20ae985 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,10 +4,9 @@ Released 2017-mm-dd - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. - Bugfixes: - Bounding box parameter ignored in static named maps #735. - + - camhaft 0.59.1 fixes duplicate columns in aggregate-intersection analysis ## 3.12.10 Released 2017-09-18 diff --git a/package.json b/package.json index a3ca339d..ba7422b4 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.59.0", + "camshaft": "0.59.1", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.3.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index 8400e973..e1ad9ba7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -219,9 +219,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.59.0: - version "0.59.0" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.0.tgz#699c0834116ec87e067d42e11eea9540bddc6383" +camshaft@0.59.1: + version "0.59.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.1.tgz#a2e87fa4a0236655160cd1a6a9e966cbbf86dd43" dependencies: async "^1.5.2" bunyan "1.8.1" From 78b75c7a88b0890c145889528cbc48f3647ee152 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 28 Sep 2017 10:47:44 +0000 Subject: [PATCH 146/177] Removes `list` dataview type --- NEWS.md | 7 +- docs/MapConfig-Dataviews-extension.md | 41 +------ lib/cartodb/models/dataview/index.js | 3 +- lib/cartodb/models/dataview/list.js | 66 ----------- .../models/dataview/overviews/index.js | 3 +- lib/cartodb/models/dataview/overviews/list.js | 11 -- package.json | 2 +- test/acceptance/widgets/list.js | 51 --------- test/acceptance/widgets/ported/list.js | 106 ------------------ .../dataviews-widgets-adapter.test.js | 73 ------------ 10 files changed, 11 insertions(+), 352 deletions(-) delete mode 100644 lib/cartodb/models/dataview/list.js delete mode 100644 lib/cartodb/models/dataview/overviews/list.js delete mode 100644 test/acceptance/widgets/list.js delete mode 100644 test/acceptance/widgets/ported/list.js diff --git a/NEWS.md b/NEWS.md index c20ae985..f0ea0728 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,12 @@ # Changelog -## 3.13.0 +## 4.0.0 Released 2017-mm-dd + +Backward incompatible changes: + - Removes `list` dataview type. + +Announcements: - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. Bugfixes: diff --git a/docs/MapConfig-Dataviews-extension.md b/docs/MapConfig-Dataviews-extension.md index d8e23a82..c2b2b8da 100644 --- a/docs/MapConfig-Dataviews-extension.md +++ b/docs/MapConfig-Dataviews-extension.md @@ -8,50 +8,13 @@ This specification describes an extension for This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`. -It makes possible to get tabular data from analysis nodes: lists, aggregated lists, aggregations, and histograms. +It makes possible to get tabular data from analysis nodes: aggregated lists, aggregations, and histograms. ## 2.1. Dataview types -### List - -A list is a simple result set per row where is possible to retrieve several columns from the original layer query. - -Definition -``` -{ - // REQUIRED - // string, `type` the list type - “type”: “list”, - // REQUIRED - // object, `options` dataview params - “options”: { - // REQUIRED - // array, `columns` to select for the list - “columns”: [“name”, “description”] - } -} -``` - -Expected output -``` -{ - "type": "list", - "rows": [ - { - "{columnName1}": "val1", - "{columnName2}": 100 - }, - { - "{columnName1}": "val2", - "{columnName2}": 200 - } - ] -} -``` - ### Aggregation -An aggregation is very similar to a list but results are aggregated by a column and a given aggregation function. +An aggregation is a list with aggregated results by a column and a given aggregation function. Definition ``` diff --git a/lib/cartodb/models/dataview/index.js b/lib/cartodb/models/dataview/index.js index d508f1bb..05cfaab7 100644 --- a/lib/cartodb/models/dataview/index.js +++ b/lib/cartodb/models/dataview/index.js @@ -1,6 +1,5 @@ module.exports = { Aggregation: require('./aggregation'), Formula: require('./formula'), - Histogram: require('./histogram'), - List: require('./list') + Histogram: require('./histogram') }; diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js deleted file mode 100644 index 33993777..00000000 --- a/lib/cartodb/models/dataview/list.js +++ /dev/null @@ -1,66 +0,0 @@ -var dot = require('dot'); -dot.templateSettings.strip = false; - -var BaseWidget = require('./base'); - -var TYPE = 'list'; - -var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list'); - -/** -{ - type: 'list', - options: { - columns: ['name', 'description'] - } -} -*/ - -function List(query, options) { - options = options || {}; - - if (!Array.isArray(options.columns)) { - throw new Error('List expects `columns` array in widget options'); - } - - BaseWidget.apply(this); - - this.query = query; - this.columns = options.columns; -} - -List.prototype = new BaseWidget(); -List.prototype.constructor = List; - -module.exports = List; - -List.prototype.sql = function(psql, override, callback) { - if (!callback) { - callback = override; - } - - var listSql = listSqlTpl({ - _query: this.query, - _columns: this.columns.join(', ') - }); - - return callback(null, listSql); -}; - -List.prototype.format = function(result) { - return { - rows: result.rows - }; -}; - -List.prototype.getType = function() { - return TYPE; -}; - -List.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _columns: this.columns.join(', ') - }); -}; diff --git a/lib/cartodb/models/dataview/overviews/index.js b/lib/cartodb/models/dataview/overviews/index.js index c541973e..ac032749 100644 --- a/lib/cartodb/models/dataview/overviews/index.js +++ b/lib/cartodb/models/dataview/overviews/index.js @@ -1,6 +1,5 @@ module.exports = { Aggregation: require('./aggregation'), Formula: require('./formula'), - Histogram: require('./histogram'), - List: require('./list') + Histogram: require('./histogram') }; diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js deleted file mode 100644 index 6ec731f4..00000000 --- a/lib/cartodb/models/dataview/overviews/list.js +++ /dev/null @@ -1,11 +0,0 @@ -var BaseOverviewsDataview = require('./base'); -var BaseDataview = require('../list'); - -function List(query, options, queryRewriter, queryRewriteData, params, queries) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); -} - -List.prototype = Object.create(BaseOverviewsDataview.prototype); -List.prototype.constructor = List; - -module.exports = List; diff --git a/package.json b/package.json index ba7422b4..36b3f355 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.13.0", + "version": "4.0.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" diff --git a/test/acceptance/widgets/list.js b/test/acceptance/widgets/list.js deleted file mode 100644 index 87fbf7df..00000000 --- a/test/acceptance/widgets/list.js +++ /dev/null @@ -1,51 +0,0 @@ -require('../../support/test_helper'); - -var assert = require('../../support/assert'); -var TestClient = require('../../support/test-client'); - -describe('list widgets', function() { - - it("should expose layer list", function(done) { - - var listWidgetMapConfig = { - version: '1.5.0', - layers: [ - { - type: 'mapnik', - options: { - sql: 'select * from test_table', - cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', - cartocss_version: '2.3.0', - widgets: { - names: { - type: 'list', - options: { - columns: ['name'] - } - } - } - } - } - ] - }; - - var testClient = new TestClient(listWidgetMapConfig); - - testClient.getWidget('names', function(err, res) { - if (err) { - return done(err); - } - - var expectedList = [ - {name:"Hawai"}, - {name:"El Estocolmo"}, - {name:"El Rey del Tallarín"}, - {name:"El Lacón"}, - {name:"El Pico"} - ]; - assert.deepEqual(JSON.parse(res.body).rows, expectedList); - - testClient.drain(done); - }); - }); -}); diff --git a/test/acceptance/widgets/ported/list.js b/test/acceptance/widgets/ported/list.js deleted file mode 100644 index e33156f7..00000000 --- a/test/acceptance/widgets/ported/list.js +++ /dev/null @@ -1,106 +0,0 @@ -require('../../../support/test_helper'); - -var assert = require('../../../support/assert'); -var TestClient = require('../../../support/test-client'); -var _ = require('underscore'); - -describe('widgets', function() { - - describe('lists', function() { - - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - - function listsMapConfig(columns) { - return { - version: '1.5.0', - layers: [ - { - type: 'mapnik', - options: { - sql: 'select * from test_table', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', - cartocss_version: '2.0.1', - widgets: { - places: { - type: 'list', - options: { - columns: columns || ['name', 'address'] - } - } - } - } - } - ] - }; - } - - var EXPECTED_NAMES = ['Hawai', 'El Estocolmo', 'El Rey del Tallarín', 'El Lacón', 'El Pico']; - - it('can be fetched from a valid list', function(done) { - var columns = ['name', 'address']; - this.testClient = new TestClient(listsMapConfig(columns)); - this.testClient.getWidget('places', function (err, res, list) { - assert.ok(!err, err); - assert.ok(list); - assert.equal(list.type, 'list'); - assert.equal(list.rows.length, 5); - - assert.ok(onlyHasFields(list, columns)); - - var names = list.rows.map(function (item) { - return item.name; - }); - assert.deepEqual(names, EXPECTED_NAMES); - - var expectedAddresses = [ - 'Calle de Pérez Galdós 9, Madrid, Spain', - 'Calle de la Palma 72, Madrid, Spain', - 'Plaza Conde de Toreno 2, Madrid, Spain', - 'Manuel Fernández y González 8, Madrid, Spain', - 'Calle Divino Pastor 12, Madrid, Spain' - ]; - var addresses = list.rows.map(function (item) { - return item.address; - }); - assert.deepEqual(addresses, expectedAddresses); - - done(); - }); - }); - - it('should fetch just one column', function(done) { - var columns = ['name']; - this.testClient = new TestClient(listsMapConfig(columns)); - this.testClient.getWidget('places', function (err, res, list) { - assert.ok(!err, err); - assert.ok(list); - assert.equal(list.type, 'list'); - assert.equal(list.rows.length, 5); - - assert.ok(onlyHasFields(list, columns)); - - var names = list.rows.map(function (item) { - return item.name; - }); - assert.deepEqual(names, EXPECTED_NAMES); - - done(); - }); - }); - - function onlyHasFields(list, expectedFields) { - var fields = (!!list.rows[0]) ? Object.keys(list.rows[0]) : []; - - return _.difference(fields, expectedFields).length === 0 && - _.difference(expectedFields, fields).length === 0; - } - - }); - -}); diff --git a/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js b/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js index 797ce06e..b78d8a85 100644 --- a/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js +++ b/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js @@ -143,79 +143,6 @@ describe('dataviews-widgets-adapter', function() { } } }, - { - "input": { - "version": "1.4.0", - "layers": [ - { - "type": "mapnik", - "options": { - "sql": "select * from test_table", - "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }", - "cartocss_version": "2.3.0", - "widgets": { - "names": { - "type": "list", - "options": { - "columns": [ - "name" - ] - } - } - } - } - } - ] - }, - "expected": { - "version": "1.4.0", - "layers": [ - { - "type": "mapnik", - "options": { - "source": { - "id": "cdb-layer-source-0" - }, - "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }", - "cartocss_version": "2.3.0", - // keep them for now - "widgets": { - "names": { - "type": "list", - "options": { - "columns": [ - "name" - ] - } - } - } - } - } - ], - "analyses": [ - { - "id": "cdb-layer-source-0", - "type": "source", - "params": { - "query": "select * from test_table" - } - } - ], - "dataviews": { - "names": { - "source": { - "id": "cdb-layer-source-0" - }, - "type": "list", - "options": { - "columns": [ - "name" - ] - } - } - } - } - }, { "input": { "version": "1.4.0", From ff47027a514254fde710c3123257bc43a83ef3de Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Thu, 28 Sep 2017 17:20:07 +0200 Subject: [PATCH 147/177] upgrade express version to 4.15.5 --- package.json | 2 +- yarn.lock | 450 ++++++++++++++++----------------------------------- 2 files changed, 144 insertions(+), 308 deletions(-) diff --git a/package.json b/package.json index ba7422b4..f4056017 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "cartodb-redis": "0.14.0", "debug": "~2.2.0", "dot": "~1.0.2", - "express": "~4.13.3", + "express": "^4.15.5", "fastly-purge": "~1.0.1", "log4js": "cartodb/log4js-node#cdb", "lru-cache": "2.6.5", diff --git a/yarn.lock b/yarn.lock index e1ad9ba7..635f20e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,20 +10,16 @@ mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" - -abbrev@1.0.x: +abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" -accepts@~1.2.12: - version "1.2.13" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.2.13.tgz#e5f1f3928c6d95fd96558c36ec3d9d0de4a6ecea" +accepts@~1.3.3: + version "1.3.4" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f" dependencies: - mime-types "~2.1.6" - negotiator "0.5.3" + mime-types "~2.1.16" + negotiator "0.6.1" ajv@^4.9.1: version "4.11.8" @@ -32,15 +28,6 @@ ajv@^4.9.1: co "^4.6.0" json-stable-stringify "^1.0.1" -ajv@^5.1.0: - version "5.2.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.2.tgz#47c68d69e86f5d953103b0074a9430dc63da5e39" - dependencies: - co "^4.6.0" - fast-deep-equal "^1.0.0" - json-schema-traverse "^0.3.0" - json-stable-stringify "^1.0.1" - align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -114,11 +101,7 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - -aws4@^1.2.1, aws4@^1.6.0: +aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" @@ -163,18 +146,6 @@ boom@2.x.x: dependencies: hoek "2.x.x" -boom@4.x.x: - version "4.3.1" - resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" - dependencies: - hoek "4.x.x" - -boom@5.x.x: - version "5.2.0" - resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" - dependencies: - hoek "4.x.x" - brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -230,7 +201,7 @@ camshaft@0.59.1: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -248,15 +219,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -368,16 +339,12 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0: +commander@2.9.0, commander@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" -commander@^2.9.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" - concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -392,11 +359,11 @@ console-control-strings@^1.0.0, console-control-strings@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" -content-disposition@0.5.1: - version "0.5.1" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.1.tgz#87476c6a67c8daa87e32e87616df883ba7fb071b" +content-disposition@0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" -content-type@~1.0.1: +content-type@~1.0.1, content-type@~1.0.2: version "1.0.4" resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" @@ -404,9 +371,9 @@ cookie-signature@1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" -cookie@0.1.5: - version "0.1.5" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.1.5.tgz#6ab9948a4b1ae21952cd2588530a4722d4044d7c" +cookie@0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.3.1.tgz#e7e0a1f9ef43b4c8ba925c5c5a96e806d16873bb" core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" @@ -418,12 +385,6 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" -cryptiles@3.x.x: - version "3.1.2" - resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" - dependencies: - boom "5.x.x" - d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -438,7 +399,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@~2.2.0: +debug@2.2.0, debug@^2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -450,15 +411,15 @@ debug@2.6.0: dependencies: ms "0.7.2" -debug@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" +debug@2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: ms "2.0.0" -debug@^2.2.0: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" +debug@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" dependencies: ms "2.0.0" @@ -492,7 +453,7 @@ delegates@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" -depd@~1.1.0: +depd@1.1.1, depd@~1.1.0, depd@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" @@ -532,11 +493,7 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" - -dot@~1.0.2: +dot@^1.0.3, dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -556,6 +513,10 @@ ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" +encodeurl@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.1.tgz#79e3d58655346909fe6f0f45a5de68103b294d20" + entities@1.0: version "1.0.0" resolved "https://registry.yarnpkg.com/entities/-/entities-1.0.0.tgz#b2987aa3821347fcde642b24fdfc9e4fb712bf26" @@ -609,45 +570,48 @@ esutils@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b" -etag@~1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/etag/-/etag-1.7.0.tgz#03d30b5f67dd6e632d2945d30d6652731a34d5d8" +etag@~1.8.0, etag@~1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" exit@0.1.2, exit@0.1.x: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" -express@~4.13.3: - version "4.13.4" - resolved "https://registry.yarnpkg.com/express/-/express-4.13.4.tgz#3c0b76f3c77590c8345739061ec0bd3ba067ec24" +express@^4.15.5: + version "4.15.5" + resolved "https://registry.yarnpkg.com/express/-/express-4.15.5.tgz#670235ca9598890a5ae8170b83db722b842ed927" dependencies: - accepts "~1.2.12" + accepts "~1.3.3" array-flatten "1.1.1" - content-disposition "0.5.1" - content-type "~1.0.1" - cookie "0.1.5" + content-disposition "0.5.2" + content-type "~1.0.2" + cookie "0.3.1" cookie-signature "1.0.6" - debug "~2.2.0" - depd "~1.1.0" + debug "2.6.9" + depd "~1.1.1" + encodeurl "~1.0.1" escape-html "~1.0.3" - etag "~1.7.0" - finalhandler "0.4.1" - fresh "0.3.0" + etag "~1.8.0" + finalhandler "~1.0.6" + fresh "0.5.2" merge-descriptors "1.0.1" methods "~1.1.2" on-finished "~2.3.0" parseurl "~1.3.1" path-to-regexp "0.1.7" - proxy-addr "~1.0.10" - qs "4.0.0" - range-parser "~1.0.3" - send "0.13.1" - serve-static "~1.10.2" - type-is "~1.6.6" + proxy-addr "~1.1.5" + qs "6.5.0" + range-parser "~1.2.0" + send "0.15.6" + serve-static "1.12.6" + setprototypeof "1.0.3" + statuses "~1.3.1" + type-is "~1.6.15" utils-merge "1.0.0" - vary "~1.0.1" + vary "~1.1.1" -extend@~3.0.0, extend@~3.0.1: +extend@~3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -655,10 +619,6 @@ extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" -fast-deep-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" - fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -669,13 +629,16 @@ fastly-purge@~1.0.1: dependencies: request "^2.55.0" -finalhandler@0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-0.4.1.tgz#85a17c6c59a94717d262d61230d4b0ebe3d4a14d" +finalhandler@~1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.0.6.tgz#007aea33d1a4d3e42017f624848ad58d212f814f" dependencies: - debug "~2.2.0" + debug "2.6.9" + encodeurl "~1.0.1" escape-html "~1.0.3" on-finished "~2.3.0" + parseurl "~1.3.2" + statuses "~1.3.1" unpipe "~1.0.0" find-up@^1.0.0: @@ -697,21 +660,13 @@ form-data@~2.1.1: combined-stream "^1.0.5" mime-types "^2.1.12" -form-data@~2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.5" - mime-types "^2.1.12" - forwarded@~0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" -fresh@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.3.0.tgz#651f838e22424e7566de161d8358caa199f83d4f" +fresh@0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" fs.realpath@^1.0.0: version "1.0.0" @@ -790,7 +745,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1: +glob@7.1.1, glob@^7.0.5, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -821,17 +776,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -872,10 +816,6 @@ har-schema@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - har-validator@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d" @@ -892,13 +832,6 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" -har-validator@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" - dependencies: - ajv "^5.1.0" - har-schema "^2.0.0" - has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -922,15 +855,6 @@ hawk@3.1.3, hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" -hawk@~6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" - dependencies: - boom "4.x.x" - cryptiles "3.x.x" - hoek "4.x.x" - sntp "2.x.x" - hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -942,10 +866,6 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" -hoek@4.x.x: - version "4.2.0" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" - hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" @@ -967,6 +887,15 @@ http-errors@~1.3.1: inherits "~2.0.1" statuses "1" +http-errors@~1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.2.tgz#0a002cc85707192a7e7946ceedc11155f60ec736" + dependencies: + depd "1.1.1" + inherits "2.0.3" + setprototypeof "1.0.3" + statuses ">= 1.3.1 < 2" + http-signature@~1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf" @@ -975,14 +904,6 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -998,7 +919,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -1010,9 +931,9 @@ invert-kv@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" -ipaddr.js@1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.0.5.tgz#5fa78cf301b825c78abc3042d812723049ea23c7" +ipaddr.js@1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.4.0.tgz#296aca878a821816e5b85d0a285a99bcff4582f0" is-arrayish@^0.2.1: version "0.2.1" @@ -1122,10 +1043,6 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" -json-schema-traverse@^0.3.0: - version "0.3.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" - json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -1339,13 +1256,13 @@ mime-db@~1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, mime-types@~2.1.7: +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.7: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: mime-db "~1.30.0" -mime@1.3.4: +mime@1.3.4, mime@~1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1353,17 +1270,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -mime@~1.3.4: - version "1.3.6" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" - -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8: +minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1371,10 +1284,6 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -minimist@~0.0.1: - version "0.0.10" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" - minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1441,9 +1350,9 @@ ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" -negotiator@0.5.3: - version "0.5.3" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.5.3.tgz#269d5c476810ec92edbe7b6c2f28316384f9a7e8" +negotiator@0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" nock@~2.11.0: version "2.11.0" @@ -1510,7 +1419,7 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1, oauth-sign@~0.8.2: +oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" @@ -1587,7 +1496,7 @@ parse-json@^2.2.0: dependencies: error-ex "^1.2.0" -parseurl@~1.3.1: +parseurl@~1.3.1, parseurl@~1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" @@ -1617,10 +1526,6 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -1641,7 +1546,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1749,25 +1654,25 @@ protozero@~1.4.2: version "1.4.5" resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.4.5.tgz#80eaa80a4f9c751465c4cb2620d8233b50ec1aff" -proxy-addr@~1.0.10: - version "1.0.10" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-1.0.10.tgz#0d40a82f801fc355567d2ecb65efe3f077f121c5" +proxy-addr@~1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-1.1.5.tgz#71c0ee3b102de3f202f3b64f608d173fcba1a918" dependencies: forwarded "~0.1.0" - ipaddr.js "1.0.5" + ipaddr.js "1.4.0" punycode@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" -qs@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-4.0.0.tgz#c31d9b74ec27df75e543a86c78728ed8d4623607" - qs@5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/qs/-/qs-5.2.0.tgz#a9f31142af468cb72b25b30136ba2456834916be" +qs@6.5.0: + version "6.5.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.0.tgz#8d04954d364def3efc55b5a0793e1e2c8b1e6e49" + qs@~6.3.0: version "6.3.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.3.2.tgz#e75bd5f6e268122a2a0e0bda630b2550c166502c" @@ -1776,17 +1681,13 @@ qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" -qs@~6.5.1: - version "6.5.1" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" - queue-async@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" -range-parser@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.0.3.tgz#6872823535c692e2c2a0103826afd82c2e0ff175" +range-parser@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e" raw-body@~2.1.5: version "2.1.7" @@ -1820,7 +1721,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1: +readable-stream@1.1, readable-stream@~1.1.9: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1850,15 +1751,6 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@~1.1.9: - version "1.1.14" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1903,34 +1795,7 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@2.x, request@^2.55.0, request@^2.69.0: - version "2.82.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.82.0.tgz#2ba8a92cd7ac45660ea2b10a53ae67cd247516ea" - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.6.0" - caseless "~0.12.0" - combined-stream "~1.0.5" - extend "~3.0.1" - forever-agent "~0.6.1" - form-data "~2.3.1" - har-validator "~5.0.3" - hawk "~6.0.2" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.17" - oauth-sign "~0.8.2" - performance-now "^2.1.0" - qs "~6.5.1" - safe-buffer "^5.1.1" - stringstream "~0.0.5" - tough-cookie "~2.3.2" - tunnel-agent "^0.6.0" - uuid "^3.1.0" - -request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1985,7 +1850,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" @@ -1993,9 +1858,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" @@ -2009,56 +1874,41 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - -send@0.13.1: - version "0.13.1" - resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" +send@0.15.6: + version "0.15.6" + resolved "https://registry.yarnpkg.com/send/-/send-0.15.6.tgz#20f23a9c925b762ab82705fe2f9db252ace47e34" dependencies: - debug "~2.2.0" - depd "~1.1.0" + debug "2.6.9" + depd "~1.1.1" destroy "~1.0.4" + encodeurl "~1.0.1" escape-html "~1.0.3" - etag "~1.7.0" - fresh "0.3.0" - http-errors "~1.3.1" + etag "~1.8.1" + fresh "0.5.2" + http-errors "~1.6.2" mime "1.3.4" - ms "0.7.1" + ms "2.0.0" on-finished "~2.3.0" - range-parser "~1.0.3" - statuses "~1.2.1" + range-parser "~1.2.0" + statuses "~1.3.1" -send@0.13.2: - version "0.13.2" - resolved "https://registry.yarnpkg.com/send/-/send-0.13.2.tgz#765e7607c8055452bba6f0b052595350986036de" +serve-static@1.12.6: + version "1.12.6" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.12.6.tgz#b973773f63449934da54e5beba5e31d9f4211577" dependencies: - debug "~2.2.0" - depd "~1.1.0" - destroy "~1.0.4" + encodeurl "~1.0.1" escape-html "~1.0.3" - etag "~1.7.0" - fresh "0.3.0" - http-errors "~1.3.1" - mime "1.3.4" - ms "0.7.1" - on-finished "~2.3.0" - range-parser "~1.0.3" - statuses "~1.2.1" - -serve-static@~1.10.2: - version "1.10.3" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.10.3.tgz#ce5a6ecd3101fed5ec09827dac22a9c29bfb0535" - dependencies: - escape-html "~1.0.3" - parseurl "~1.3.1" - send "0.13.2" + parseurl "~1.3.2" + send "0.15.6" set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" +setprototypeof@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.0.3.tgz#66567e37043eeb4f04d91bd658c0cbefb55b8e04" + shelljs@0.3.x: version "0.3.0" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" @@ -2077,12 +1927,6 @@ sntp@1.x.x: dependencies: hoek "2.x.x" -sntp@2.x.x: - version "2.0.2" - resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" - dependencies: - hoek "4.x.x" - source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -2117,14 +1961,10 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4: +sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" -sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" - split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2162,14 +2002,10 @@ sshpk@^1.7.0: jsbn "~0.1.0" tweetnacl "~0.14.0" -statuses@1: +statuses@1, "statuses@>= 1.3.1 < 2", statuses@~1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" -statuses@~1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.2.1.tgz#dded45cc18256d51ed40aec142489d5c61026d28" - step-profiler@~0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/step-profiler/-/step-profiler-0.3.0.tgz#841368ce44f2330c862edd5ec60e7fbec148f0b3" @@ -2202,7 +2038,7 @@ string_decoder@~1.0.3: dependencies: safe-buffer "~5.1.0" -stringstream@~0.0.4, stringstream@~0.0.5: +stringstream@~0.0.4: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2274,7 +2110,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2282,7 +2118,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2307,7 +2143,7 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0, tough-cookie@~2.3.2: +tough-cookie@~2.3.0: version "2.3.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: @@ -2352,7 +2188,7 @@ type-detect@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" -type-is@~1.6.10, type-is@~1.6.6: +type-is@~1.6.10, type-is@~1.6.15: version "1.6.15" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410" dependencies: @@ -2400,7 +2236,7 @@ utils-merge@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" -uuid@^3.0.0, uuid@^3.1.0: +uuid@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" @@ -2411,9 +2247,9 @@ validate-npm-package-license@^3.0.1: spdx-correct "~1.0.0" spdx-expression-parse "~1.0.0" -vary@~1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/vary/-/vary-1.0.1.tgz#99e4981566a286118dfb2b817357df7993376d10" +vary@~1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" verror@1.10.0: version "1.10.0" From 6149df1810c29fb7ca7d1e3de07e37b79546ef64 Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Thu, 28 Sep 2017 17:37:43 +0200 Subject: [PATCH 148/177] upgrade body-parser to version 1.17.2 version 1.17.2 requires a patched query string (qs) module --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f4056017..776f3709 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "Mario de Frutos " ], "dependencies": { - "body-parser": "~1.14.0", + "body-parser": "~1.17.2", "camshaft": "0.59.1", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.3.0", From aaf95b9223af9aa5cdd915dedbbe83528fb10964 Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Thu, 28 Sep 2017 17:41:49 +0200 Subject: [PATCH 149/177] upgrade body-parser (package.json and yarn.lock) --- package.json | 2 +- yarn.lock | 76 +++++++++++++++++++++++----------------------------- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/package.json b/package.json index 776f3709..5f025bb5 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "Mario de Frutos " ], "dependencies": { - "body-parser": "~1.17.2", + "body-parser": "^1.18.2", "camshaft": "0.59.1", "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.3.0", diff --git a/yarn.lock b/yarn.lock index 635f20e7..6ec6a03c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -125,20 +125,20 @@ block-stream@*: dependencies: inherits "~2.0.0" -body-parser@~1.14.0: - version "1.14.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.14.2.tgz#1015cb1fe2c443858259581db53332f8d0cf50f9" +body-parser@^1.18.2: + version "1.18.2" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.2.tgz#87678a19d84b47d859b83199bd59bce222b10454" dependencies: - bytes "2.2.0" - content-type "~1.0.1" - debug "~2.2.0" - depd "~1.1.0" - http-errors "~1.3.1" - iconv-lite "0.4.13" + bytes "3.0.0" + content-type "~1.0.4" + debug "2.6.9" + depd "~1.1.1" + http-errors "~1.6.2" + iconv-lite "0.4.19" on-finished "~2.3.0" - qs "5.2.0" - raw-body "~2.1.5" - type-is "~1.6.10" + qs "6.5.1" + raw-body "2.3.2" + type-is "~1.6.15" boom@2.x.x: version "2.10.1" @@ -174,13 +174,9 @@ bunyan@1.8.1: mv "~2" safe-json-stringify "~1" -bytes@2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-2.2.0.tgz#fd35464a403f6f9117c2de3609ecff9cae000588" - -bytes@2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-2.4.0.tgz#7d97196f9d5baf7f6935e25985549edd2a6c2339" +bytes@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" camelcase@^1.0.2: version "1.2.1" @@ -363,7 +359,7 @@ content-disposition@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" -content-type@~1.0.1, content-type@~1.0.2: +content-type@~1.0.2, content-type@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" @@ -453,7 +449,7 @@ delegates@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" -depd@1.1.1, depd@~1.1.0, depd@~1.1.1: +depd@1.1.1, depd@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" @@ -880,14 +876,7 @@ htmlparser2@3.8.x: entities "1.0" readable-stream "1.1" -http-errors@~1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.3.1.tgz#197e22cdebd4198585e8694ef6786197b91ed942" - dependencies: - inherits "~2.0.1" - statuses "1" - -http-errors@~1.6.2: +http-errors@1.6.2, http-errors@~1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.2.tgz#0a002cc85707192a7e7946ceedc11155f60ec736" dependencies: @@ -908,9 +897,9 @@ husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" -iconv-lite@0.4.13: - version "0.4.13" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.13.tgz#1f88aba4ab0b1508e8312acc39345f36e992e2f2" +iconv-lite@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" inflight@^1.0.4: version "1.0.6" @@ -1665,14 +1654,14 @@ punycode@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" -qs@5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-5.2.0.tgz#a9f31142af468cb72b25b30136ba2456834916be" - qs@6.5.0: version "6.5.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.0.tgz#8d04954d364def3efc55b5a0793e1e2c8b1e6e49" +qs@6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + qs@~6.3.0: version "6.3.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.3.2.tgz#e75bd5f6e268122a2a0e0bda630b2550c166502c" @@ -1689,12 +1678,13 @@ range-parser@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e" -raw-body@~2.1.5: - version "2.1.7" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.1.7.tgz#adfeace2e4fb3098058014d08c072dcc59758774" +raw-body@2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.2.tgz#bcd60c77d3eb93cde0050295c3f379389bc88f89" dependencies: - bytes "2.4.0" - iconv-lite "0.4.13" + bytes "3.0.0" + http-errors "1.6.2" + iconv-lite "0.4.19" unpipe "1.0.0" rc@^1.1.7: @@ -2002,7 +1992,7 @@ sshpk@^1.7.0: jsbn "~0.1.0" tweetnacl "~0.14.0" -statuses@1, "statuses@>= 1.3.1 < 2", statuses@~1.3.1: +"statuses@>= 1.3.1 < 2", statuses@~1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" @@ -2188,7 +2178,7 @@ type-detect@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" -type-is@~1.6.10, type-is@~1.6.15: +type-is@~1.6.15: version "1.6.15" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410" dependencies: From cacb92b0c4aa40b962b6fe7ac11580d94d251afe Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Thu, 28 Sep 2017 18:24:50 +0200 Subject: [PATCH 150/177] upgrade request module --- package.json | 2 +- yarn.lock | 125 ++++++++++++++++++++------------------------------- 2 files changed, 50 insertions(+), 77 deletions(-) diff --git a/package.json b/package.json index ba7422b4..58bc500d 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "node-statsd": "~0.0.7", "queue-async": "~1.0.7", "redis-mpool": "0.4.1", - "request": "~2.79.0", + "request": "^2.83.0", "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", diff --git a/yarn.lock b/yarn.lock index e1ad9ba7..25a7fa2c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,11 +10,7 @@ mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" - -abbrev@1.0.x: +abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -230,7 +226,7 @@ camshaft@0.59.1: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -248,15 +244,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -368,16 +364,12 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0: +commander@2.9.0, commander@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" -commander@^2.9.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" - concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -438,7 +430,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@~2.2.0: +debug@2.2.0, debug@^2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -456,12 +448,6 @@ debug@^1.0.4: dependencies: ms "2.0.0" -debug@^2.2.0: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - dependencies: - ms "2.0.0" - decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -532,11 +518,7 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" - -dot@~1.0.2: +dot@^1.0.3, dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -790,7 +772,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1: +glob@7.1.1, glob@^7.0.5, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -821,17 +803,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1345,7 +1316,7 @@ mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, m dependencies: mime-db "~1.30.0" -mime@1.3.4: +mime@1.3.4, mime@~1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1353,17 +1324,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -mime@~1.3.4: - version "1.3.6" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" - -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8: +minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1371,10 +1338,6 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -minimist@~0.0.1: - version "0.0.10" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" - minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1641,7 +1604,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1820,7 +1783,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1: +readable-stream@1.1, readable-stream@~1.1.9: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1850,15 +1813,6 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@~1.1.9: - version "1.1.14" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1930,6 +1884,33 @@ request@2.x, request@^2.55.0, request@^2.69.0: tunnel-agent "^0.6.0" uuid "^3.1.0" +request@^2.83.0: + version "2.83.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + hawk "~6.0.2" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + stringstream "~0.0.5" + tough-cookie "~2.3.3" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" @@ -1993,9 +1974,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" @@ -2009,10 +1990,6 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - send@0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" @@ -2117,14 +2094,10 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4: +sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" -sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" - split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2274,7 +2247,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2282,7 +2255,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2307,7 +2280,7 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0, tough-cookie@~2.3.2: +tough-cookie@~2.3.0, tough-cookie@~2.3.2, tough-cookie@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: From 52402c033304d842eea24038789e949e27d0bd61 Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Thu, 28 Sep 2017 18:25:40 +0200 Subject: [PATCH 151/177] Revert "upgrade request module" This reverts commit cacb92b0c4aa40b962b6fe7ac11580d94d251afe. --- package.json | 2 +- yarn.lock | 125 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 77 insertions(+), 50 deletions(-) diff --git a/package.json b/package.json index 58bc500d..ba7422b4 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "node-statsd": "~0.0.7", "queue-async": "~1.0.7", "redis-mpool": "0.4.1", - "request": "^2.83.0", + "request": "~2.79.0", "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", diff --git a/yarn.lock b/yarn.lock index 25a7fa2c..e1ad9ba7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -226,7 +230,7 @@ camshaft@0.59.1: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -244,15 +248,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -364,12 +368,16 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0, commander@^2.9.0: +commander@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" +commander@^2.9.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -430,7 +438,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@^2.2.0, debug@~2.2.0: +debug@2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -448,6 +456,12 @@ debug@^1.0.4: dependencies: ms "2.0.0" +debug@^2.2.0: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + dependencies: + ms "2.0.0" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -518,7 +532,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -772,7 +790,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -803,6 +821,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1316,7 +1345,7 @@ mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, m dependencies: mime-db "~1.30.0" -mime@1.3.4, mime@~1.3.4: +mime@1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1324,13 +1353,17 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +mime@~1.3.4: + version "1.3.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" + +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1338,6 +1371,10 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1604,7 +1641,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: +"pg@github:cartodb/node-postgres#6.1.6-cdb1": version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1783,7 +1820,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1813,6 +1850,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1884,33 +1930,6 @@ request@2.x, request@^2.55.0, request@^2.69.0: tunnel-agent "^0.6.0" uuid "^3.1.0" -request@^2.83.0: - version "2.83.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.6.0" - caseless "~0.12.0" - combined-stream "~1.0.5" - extend "~3.0.1" - forever-agent "~0.6.1" - form-data "~2.3.1" - har-validator "~5.0.3" - hawk "~6.0.2" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.17" - oauth-sign "~0.8.2" - performance-now "^2.1.0" - qs "~6.5.1" - safe-buffer "^5.1.1" - stringstream "~0.0.5" - tough-cookie "~2.3.3" - tunnel-agent "^0.6.0" - uuid "^3.1.0" - request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" @@ -1974,9 +1993,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" semver@4.3.2: version "4.3.2" @@ -1990,6 +2009,10 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + send@0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" @@ -2094,10 +2117,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2247,7 +2274,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2255,7 +2282,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2280,7 +2307,7 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0, tough-cookie@~2.3.2, tough-cookie@~2.3.3: +tough-cookie@~2.3.0, tough-cookie@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: From 649383a3df1af7b45a97b78f2b2a80bf63fc71b4 Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Thu, 28 Sep 2017 18:28:04 +0200 Subject: [PATCH 152/177] upgrade request module to version 2.83.0 --- package.json | 2 +- yarn.lock | 138 +++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 5f025bb5..b622c7ba 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "node-statsd": "~0.0.7", "queue-async": "~1.0.7", "redis-mpool": "0.4.1", - "request": "~2.79.0", + "request": "^2.83.0", "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", diff --git a/yarn.lock b/yarn.lock index 6ec6a03c..49fcb9c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,6 +28,15 @@ ajv@^4.9.1: co "^4.6.0" json-stable-stringify "^1.0.1" +ajv@^5.1.0: + version "5.2.3" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.3.tgz#c06f598778c44c6b161abafe3466b81ad1814ed2" + dependencies: + co "^4.6.0" + fast-deep-equal "^1.0.0" + json-schema-traverse "^0.3.0" + json-stable-stringify "^1.0.1" + align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -101,7 +110,11 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws4@^1.2.1: +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + +aws4@^1.2.1, aws4@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" @@ -146,6 +159,18 @@ boom@2.x.x: dependencies: hoek "2.x.x" +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" + dependencies: + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + dependencies: + hoek "4.x.x" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -381,6 +406,12 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" +cryptiles@3.x.x: + version "3.1.2" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" + dependencies: + boom "5.x.x" + d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -395,7 +426,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@^2.2.0, debug@~2.2.0: +debug@2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -407,7 +438,7 @@ debug@2.6.0: dependencies: ms "0.7.2" -debug@2.6.9: +debug@2.6.9, debug@^2.2.0: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: @@ -607,7 +638,7 @@ express@^4.15.5: utils-merge "1.0.0" vary "~1.1.1" -extend@~3.0.0: +extend@~3.0.0, extend@~3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -615,6 +646,10 @@ extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" +fast-deep-equal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" + fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -656,6 +691,14 @@ form-data@~2.1.1: combined-stream "^1.0.5" mime-types "^2.1.12" +form-data@~2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.5" + mime-types "^2.1.12" + forwarded@~0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" @@ -812,6 +855,10 @@ har-schema@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + har-validator@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d" @@ -828,6 +875,13 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" +har-validator@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" + dependencies: + ajv "^5.1.0" + har-schema "^2.0.0" + has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -851,6 +905,15 @@ hawk@3.1.3, hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" +hawk@~6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -862,6 +925,10 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" +hoek@4.x.x: + version "4.2.0" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" + hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" @@ -893,6 +960,14 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -1032,6 +1107,10 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" +json-schema-traverse@^0.3.0: + version "0.3.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" + json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -1245,7 +1324,7 @@ mime-db@~1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.7: +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17, mime-types@~2.1.7: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: @@ -1408,7 +1487,7 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1: +oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" @@ -1515,6 +1594,10 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -1658,7 +1741,7 @@ qs@6.5.0: version "6.5.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.0.tgz#8d04954d364def3efc55b5a0793e1e2c8b1e6e49" -qs@6.5.1: +qs@6.5.1, qs@~6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" @@ -1810,6 +1893,33 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" +request@^2.83.0: + version "2.83.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + hawk "~6.0.2" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + stringstream "~0.0.5" + tough-cookie "~2.3.3" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1840,7 +1950,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" @@ -1917,6 +2027,12 @@ sntp@1.x.x: dependencies: hoek "2.x.x" +sntp@2.x.x: + version "2.0.2" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" + dependencies: + hoek "4.x.x" + source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -2028,7 +2144,7 @@ string_decoder@~1.0.3: dependencies: safe-buffer "~5.1.0" -stringstream@~0.0.4: +stringstream@~0.0.4, stringstream@~0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2133,7 +2249,7 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0: +tough-cookie@~2.3.0, tough-cookie@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: @@ -2226,7 +2342,7 @@ utils-merge@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" -uuid@^3.0.0: +uuid@^3.0.0, uuid@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" From 75c8a7342387ad079e83d6df9adb24f72fcf1603 Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Fri, 29 Sep 2017 10:42:20 +0200 Subject: [PATCH 153/177] upgrade debug to version 3.1 --- package.json | 2 +- yarn.lock | 58 +++++++++++++++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index b622c7ba..4370b036 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "cartodb-psql": "0.10.1", "cartodb-query-tables": "0.3.0", "cartodb-redis": "0.14.0", - "debug": "~2.2.0", + "debug": "^3.1.0", "dot": "~1.0.2", "express": "^4.15.5", "fastly-purge": "~1.0.1", diff --git a/yarn.lock b/yarn.lock index 49fcb9c0..a9eccf5d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -450,6 +450,12 @@ debug@^1.0.4: dependencies: ms "2.0.0" +debug@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + dependencies: + ms "2.0.0" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1868,32 +1874,7 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.83.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@^2.83.0: version "2.83.0" resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" dependencies: @@ -1920,6 +1901,31 @@ request@^2.83.0: tunnel-agent "^0.6.0" uuid "^3.1.0" +request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" From 818cdbd99be9b5cc9a1806dbfb62e20949b789a9 Mon Sep 17 00:00:00 2001 From: Eneko Lakasta Date: Fri, 29 Sep 2017 17:41:57 +0200 Subject: [PATCH 154/177] upgrade express again (4.16.0) --- NEWS.md | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index c20ae985..a5d3e7d3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,7 @@ ## 3.13.0 Released 2017-mm-dd - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. + - Upgrades express, debug, request and body-parser for sequrity reasons Bugfixes: - Bounding box parameter ignored in static named maps #735. diff --git a/package.json b/package.json index 4370b036..39aedfa6 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "cartodb-redis": "0.14.0", "debug": "^3.1.0", "dot": "~1.0.2", - "express": "^4.15.5", + "express": "~4.16.0", "fastly-purge": "~1.0.1", "log4js": "cartodb/log4js-node#cdb", "lru-cache": "2.6.5", From 45dece65f2677f015c9c05dde2177a28853a1590 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 2 Oct 2017 14:45:43 +0200 Subject: [PATCH 155/177] Update 3.13.0 release date in NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index c20ae985..452f80dc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 3.13.0 -Released 2017-mm-dd +Released 2017-10-02 - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. Bugfixes: From b8783a6447a08e126d926399b92c10efed29d8d8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 2 Oct 2017 14:49:01 +0200 Subject: [PATCH 156/177] Stub NEWS/package for next version --- NEWS.md | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 452f80dc..182707e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 3.13.1 +Released 2017-mm-dd + +Bugfixes: + - + ## 3.13.0 Released 2017-10-02 - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. diff --git a/package.json b/package.json index ba7422b4..c533b40e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.13.0", + "version": "3.13.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 0a7506e4b2aac2dee3e382bc8331870db976c1e8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Oct 2017 12:10:43 +0200 Subject: [PATCH 157/177] Update yarn version to v0.27.5 --- INSTALL.md | 2 +- NEWS.md | 1 + package.json | 2 +- yarn.lock | 16 ++++++++-------- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 2de2cb09..11cf7db8 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are - Core - Node.js >=6.9.x - - yarn >=0.21.3 + - yarn >=0.27.5 - PostgreSQL >8.3.x, PostGIS >1.5.x - Redis >2.4.0 (http://www.redis.io) - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik). diff --git a/NEWS.md b/NEWS.md index 182707e5..139c9a3f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## 3.13.1 Released 2017-mm-dd + - Upgrades yarn minimum version requirement to v0.27.5 Bugfixes: - diff --git a/package.json b/package.json index c533b40e..6532a04e 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,6 @@ }, "engines": { "node": ">=6.9", - "yarn": "^0.21.3" + "yarn": "^0.27.5" } } diff --git a/yarn.lock b/yarn.lock index e1ad9ba7..62fc9315 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -230,7 +230,7 @@ camshaft@0.59.1: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -248,15 +248,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1641,7 +1641,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -2274,7 +2274,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2282,7 +2282,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: From 89172f280f6032a7f3c3b14a9c99b847902c325d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Oct 2017 12:43:09 +0200 Subject: [PATCH 158/177] Change yarn dep version range specification In theory the caret `^` should work but as rochoa pointed out it does not. So changing it (also for the sake of clarity). --- INSTALL.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 11cf7db8..45654b0b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are - Core - Node.js >=6.9.x - - yarn >=0.27.5 + - yarn >=0.27.5 <1.0.0 - PostgreSQL >8.3.x, PostGIS >1.5.x - Redis >2.4.0 (http://www.redis.io) - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik). diff --git a/package.json b/package.json index 6532a04e..fd857e96 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,6 @@ }, "engines": { "node": ">=6.9", - "yarn": "^0.27.5" + "yarn": ">=0.27.5 <1.0.0" } } From 69bd14793ff15b80c96820a0997dbc69147bf316 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 3 Oct 2017 11:20:39 +0000 Subject: [PATCH 159/177] Upgrade to windshaft 3.3.3 Regenerate yarn.lock file --- NEWS.md | 1 + package.json | 2 +- yarn.lock | 277 +++++++++++++++++++++++---------------------------- 3 files changed, 127 insertions(+), 153 deletions(-) diff --git a/NEWS.md b/NEWS.md index 7c914557..54f1b915 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## 3.13.1 Released 2017-mm-dd + - Upgrades windshaft to [3.3.3](https://github.com/CartoDB/windshaft/releases/tag/3.3.3). - Upgrades yarn minimum version requirement to v0.27.5 Bugfixes: diff --git a/package.json b/package.json index 8bf90aaf..771727b4 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.0", "underscore": "~1.6.0", - "windshaft": "3.3.2", + "windshaft": "3.3.3", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index a9eccf5d..8b701f06 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,11 +10,15 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" -accepts@~1.3.3: +accepts@~1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f" dependencies: @@ -138,7 +142,7 @@ block-stream@*: dependencies: inherits "~2.0.0" -body-parser@^1.18.2: +body-parser@1.18.2, body-parser@^1.18.2: version "1.18.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.2.tgz#87678a19d84b47d859b83199bd59bce222b10454" dependencies: @@ -222,7 +226,7 @@ camshaft@0.59.1: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -240,15 +244,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -282,10 +286,6 @@ cartodb-redis@0.14.0: redis-mpool "~0.4.1" underscore "~1.6.0" -caseless@~0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.11.0.tgz#715b96ea9841593cc33067923f5ec60ebda4f7d7" - caseless@~0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" @@ -305,7 +305,7 @@ center-align@^0.1.1: deep-eql "^0.1.3" type-detect "^1.0.0" -chalk@^1.1.1, chalk@^1.1.3: +chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" dependencies: @@ -360,7 +360,7 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0, commander@^2.9.0: +commander@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: @@ -384,7 +384,7 @@ content-disposition@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" -content-type@~1.0.2, content-type@~1.0.4: +content-type@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" @@ -526,7 +526,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -603,7 +607,7 @@ esutils@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b" -etag@~1.8.0, etag@~1.8.1: +etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" @@ -611,38 +615,40 @@ exit@0.1.2, exit@0.1.x: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" -express@^4.15.5: - version "4.15.5" - resolved "https://registry.yarnpkg.com/express/-/express-4.15.5.tgz#670235ca9598890a5ae8170b83db722b842ed927" +express@~4.16.0: + version "4.16.1" + resolved "https://registry.yarnpkg.com/express/-/express-4.16.1.tgz#6b33b560183c9b253b7b62144df33a4654ac9ed0" dependencies: - accepts "~1.3.3" + accepts "~1.3.4" array-flatten "1.1.1" + body-parser "1.18.2" content-disposition "0.5.2" - content-type "~1.0.2" + content-type "~1.0.4" cookie "0.3.1" cookie-signature "1.0.6" debug "2.6.9" depd "~1.1.1" encodeurl "~1.0.1" escape-html "~1.0.3" - etag "~1.8.0" - finalhandler "~1.0.6" + etag "~1.8.1" + finalhandler "1.1.0" fresh "0.5.2" merge-descriptors "1.0.1" methods "~1.1.2" on-finished "~2.3.0" - parseurl "~1.3.1" + parseurl "~1.3.2" path-to-regexp "0.1.7" - proxy-addr "~1.1.5" - qs "6.5.0" + proxy-addr "~2.0.2" + qs "6.5.1" range-parser "~1.2.0" - send "0.15.6" - serve-static "1.12.6" - setprototypeof "1.0.3" + safe-buffer "5.1.1" + send "0.16.1" + serve-static "1.13.1" + setprototypeof "1.1.0" statuses "~1.3.1" type-is "~1.6.15" - utils-merge "1.0.0" - vary "~1.1.1" + utils-merge "1.0.1" + vary "~1.1.2" extend@~3.0.0, extend@~3.0.1: version "3.0.1" @@ -666,9 +672,9 @@ fastly-purge@~1.0.1: dependencies: request "^2.55.0" -finalhandler@~1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.0.6.tgz#007aea33d1a4d3e42017f624848ad58d212f814f" +finalhandler@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.0.tgz#ce0b6855b45853e791b2fcc680046d88253dd7f5" dependencies: debug "2.6.9" encodeurl "~1.0.1" @@ -705,7 +711,7 @@ form-data@~2.3.1: combined-stream "^1.0.5" mime-types "^2.1.12" -forwarded@~0.1.0: +forwarded@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" @@ -754,16 +760,6 @@ gdal@~0.9.2: nan "~2.6.2" node-pre-gyp "~0.6.36" -generate-function@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/generate-function/-/generate-function-2.0.0.tgz#6858fe7c0969b7d4e9093337647ac79f60dfbe74" - -generate-object-property@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/generate-object-property/-/generate-object-property-1.2.0.tgz#9c0e1c40308ce804f4783618b937fa88f99d50d0" - dependencies: - is-property "^1.0.0" - generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -790,7 +786,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -821,6 +817,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -865,15 +872,6 @@ har-schema@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" -har-validator@~2.0.6: - version "2.0.6" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d" - dependencies: - chalk "^1.1.1" - commander "^2.9.0" - is-my-json-valid "^2.12.4" - pinkie-promise "^2.0.0" - har-validator@~4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-4.2.1.tgz#33481d0f1bbff600dd203d75812a6a5fba002e2a" @@ -1001,9 +999,9 @@ invert-kv@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" -ipaddr.js@1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.4.0.tgz#296aca878a821816e5b85d0a285a99bcff4582f0" +ipaddr.js@1.5.2: + version "1.5.2" + resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.5.2.tgz#d4b505bde9946987ccf0fc58d9010ff9607e3fa0" is-arrayish@^0.2.1: version "0.2.1" @@ -1025,19 +1023,6 @@ is-fullwidth-code-point@^1.0.0: dependencies: number-is-nan "^1.0.0" -is-my-json-valid@^2.12.4: - version "2.16.1" - resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11" - dependencies: - generate-function "^2.0.0" - generate-object-property "^1.1.0" - jsonpointer "^4.0.0" - xtend "^4.0.0" - -is-property@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84" - is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -1139,10 +1124,6 @@ jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" -jsonpointer@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-4.0.1.tgz#4fd92cb34e0e9db3c89c8622ecf51f9b978c6cb9" - jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -1336,21 +1317,25 @@ mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17, dependencies: mime-db "~1.30.0" -mime@1.3.4, mime@~1.3.4: - version "1.3.4" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" +mime@1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +mime@~1.3.4: + version "1.3.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" + +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1358,6 +1343,10 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1570,7 +1559,7 @@ parse-json@^2.2.0: dependencies: error-ex "^1.2.0" -parseurl@~1.3.1, parseurl@~1.3.2: +parseurl@~1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" @@ -1624,7 +1613,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: +"pg@github:cartodb/node-postgres#6.1.6-cdb1": version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1732,29 +1721,21 @@ protozero@~1.4.2: version "1.4.5" resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.4.5.tgz#80eaa80a4f9c751465c4cb2620d8233b50ec1aff" -proxy-addr@~1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-1.1.5.tgz#71c0ee3b102de3f202f3b64f608d173fcba1a918" +proxy-addr@~2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.2.tgz#6571504f47bb988ec8180253f85dd7e14952bdec" dependencies: - forwarded "~0.1.0" - ipaddr.js "1.4.0" + forwarded "~0.1.2" + ipaddr.js "1.5.2" punycode@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" -qs@6.5.0: - version "6.5.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.0.tgz#8d04954d364def3efc55b5a0793e1e2c8b1e6e49" - qs@6.5.1, qs@~6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" -qs@~6.3.0: - version "6.3.2" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.3.2.tgz#e75bd5f6e268122a2a0e0bda630b2550c166502c" - qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" @@ -1800,7 +1781,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1830,6 +1811,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1901,31 +1891,6 @@ request@2.x, request@^2.55.0, request@^2.69.0, request@^2.83.0: tunnel-agent "^0.6.0" uuid "^3.1.0" -request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1956,7 +1921,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" @@ -1964,9 +1929,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" semver@4.3.2: version "4.3.2" @@ -1980,9 +1945,13 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -send@0.15.6: - version "0.15.6" - resolved "https://registry.yarnpkg.com/send/-/send-0.15.6.tgz#20f23a9c925b762ab82705fe2f9db252ace47e34" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + +send@0.16.1: + version "0.16.1" + resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" dependencies: debug "2.6.9" depd "~1.1.1" @@ -1992,20 +1961,20 @@ send@0.15.6: etag "~1.8.1" fresh "0.5.2" http-errors "~1.6.2" - mime "1.3.4" + mime "1.4.1" ms "2.0.0" on-finished "~2.3.0" range-parser "~1.2.0" statuses "~1.3.1" -serve-static@1.12.6: - version "1.12.6" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.12.6.tgz#b973773f63449934da54e5beba5e31d9f4211577" +serve-static@1.13.1: + version "1.13.1" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.1.tgz#4c57d53404a761d8f2e7c1e8a18a47dbf278a719" dependencies: encodeurl "~1.0.1" escape-html "~1.0.3" parseurl "~1.3.2" - send "0.15.6" + send "0.16.1" set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" @@ -2015,6 +1984,10 @@ setprototypeof@1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.0.3.tgz#66567e37043eeb4f04d91bd658c0cbefb55b8e04" +setprototypeof@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" + shelljs@0.3.x: version "0.3.0" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" @@ -2073,10 +2046,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2088,8 +2065,8 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.12" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.12.tgz#2b3a14b17162e39e8aa6e1e2487a41d0795396d8" + version "3.1.13" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.13.tgz#d990a05627392768de6278bafd1a31fdfe907dd9" dependencies: nan "~2.7.0" node-pre-gyp "~0.6.38" @@ -2222,7 +2199,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2230,7 +2207,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2267,10 +2244,6 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" -tunnel-agent@~0.4.1: - version "0.4.3" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" - turbo-carto@0.20.0: version "0.20.0" resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.0.tgz#ebeb4620f3e26ed3cce53a9ff30f087f46a2e3fd" @@ -2344,9 +2317,9 @@ util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" -utils-merge@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" +utils-merge@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" uuid@^3.0.0, uuid@^3.1.0: version "3.1.0" @@ -2359,7 +2332,7 @@ validate-npm-package-license@^3.0.1: spdx-correct "~1.0.0" spdx-expression-parse "~1.0.0" -vary@~1.1.1: +vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" @@ -2395,21 +2368,21 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.2.tgz#72efe0dbc0d8d4bcba4211fdabd15dd2e0799df9" +windshaft@3.3.3: + version "3.3.3" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.3.tgz#0582e6a0d9cf91c533134787ace64a3337200e33" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 cartodb-psql "^0.10.1" - debug "~2.2.0" + debug "^3.1.0" dot "~1.0.2" grainstore "~1.6.0" mapnik "3.5.14" queue-async "~1.0.7" redis-mpool "0.4.1" - request "~2.79.0" + request "^2.83.0" semver "~5.0.3" sphericalmercator "1.0.4" step "~0.0.6" From da832263a426906ed07d5976588d8cd999ed7dd2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 3 Oct 2017 11:46:03 +0000 Subject: [PATCH 160/177] Upgrade turbo-carto, camshaft, and cartodb-psql --- NEWS.md | 3 +++ package.json | 6 +++--- yarn.lock | 36 ++++++++++++++++++------------------ 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/NEWS.md b/NEWS.md index 54f1b915..e28c5517 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,9 @@ ## 3.13.1 Released 2017-mm-dd + - Upgrades turbo-carto to [0.20.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.1) + - Upgrades cartodb-psql to [0.10.2](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.2). + - Upgrades camshaft to [0.59.2](https://github.com/CartoDB/camshaft/releases/tag/0.59.2). - Upgrades windshaft to [3.3.3](https://github.com/CartoDB/windshaft/releases/tag/3.3.3). - Upgrades yarn minimum version requirement to v0.27.5 diff --git a/package.json b/package.json index 771727b4..f8f997b8 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,8 @@ ], "dependencies": { "body-parser": "^1.18.2", - "camshaft": "0.59.1", - "cartodb-psql": "0.10.1", + "camshaft": "0.59.2", + "cartodb-psql": "0.10.2", "cartodb-query-tables": "0.3.0", "cartodb-redis": "0.14.0", "debug": "^3.1.0", @@ -41,7 +41,7 @@ "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.20.0", + "turbo-carto": "0.20.1", "underscore": "~1.6.0", "windshaft": "3.3.3", "yargs": "~5.0.0" diff --git a/yarn.lock b/yarn.lock index 8b701f06..240bd4ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -215,14 +215,14 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.59.1: - version "0.59.1" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.1.tgz#a2e87fa4a0236655160cd1a6a9e966cbbf86dd43" +camshaft@0.59.2: + version "0.59.2" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.2.tgz#8b032771faa1264bd8a81040c6075beb1a32e286" dependencies: async "^1.5.2" bunyan "1.8.1" cartodb-psql "^0.10.1" - debug "^2.2.0" + debug "^3.1.0" dot "^1.0.3" request "^2.69.0" @@ -266,11 +266,11 @@ cartocolor@4.0.0: dependencies: colorbrewer "1.0.0" -cartodb-psql@0.10.1, cartodb-psql@^0.10.1: - version "0.10.1" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.1.tgz#0ac947e62fe10b27916df6b7ba6c461953fe3a23" +cartodb-psql@0.10.2, cartodb-psql@^0.10.1: + version "0.10.2" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.2.tgz#8c505066e4a635cfa0ee4c603769c83f6e2187dd" dependencies: - debug "~2.2.0" + debug "^3.1.0" pg cartodb/node-postgres#6.1.6-cdb1 underscore "~1.6.0" @@ -426,12 +426,6 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@~2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" - dependencies: - ms "0.7.1" - debug@2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b" @@ -456,6 +450,12 @@ debug@^3.1.0: dependencies: ms "2.0.0" +debug@~2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" + dependencies: + ms "0.7.1" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -2244,13 +2244,13 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" -turbo-carto@0.20.0: - version "0.20.0" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.0.tgz#ebeb4620f3e26ed3cce53a9ff30f087f46a2e3fd" +turbo-carto@0.20.1: + version "0.20.1" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.1.tgz#e9f5fa1408d9d4325a1e79333e6d242170f89e6d" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" - debug "2.2.0" + debug "^3.1.0" es6-promise "3.1.2" postcss "5.0.19" postcss-value-parser "3.3.0" From 8fe31c45f34b53ff1e22a1f20ebf6da66e31d5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 4 Oct 2017 11:10:17 +0200 Subject: [PATCH 161/177] fix 'this' scope with arrow function --- lib/cartodb/models/dataview/base.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index ea7f9474..8c1ada5d 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -26,7 +26,7 @@ module.exports = class BaseDataview { return callback(err); } - psql.query(query, function(err, result) { + psql.query(query, (err, result) => { if (err) { return callback(err, result); } @@ -48,7 +48,7 @@ module.exports = class BaseDataview { const readOnlyTransaction = true; const columnTypeQuery = columnTypeQueryTpl({ column, query }); - psql.query(columnTypeQuery, function(err, result) { + psql.query(columnTypeQuery, (err, result) => { if (err) { return callback(err); } From 87bcb7ebf2671892a472510ea7478cbc9b23935a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 4 Oct 2017 10:12:54 +0000 Subject: [PATCH 162/177] Update list of upgraded deps --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index c028cf01..86febfb0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,9 @@ Released 2017-mm-dd Announcements: + - Upgrades express to 4.16.0. + - Upgrades debug to 3.1.0. + - Upgrades request to 2.83.0. - Upgrades turbo-carto to [0.20.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.1) - Upgrades cartodb-psql to [0.10.2](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.2). - Upgrades camshaft to [0.59.2](https://github.com/CartoDB/camshaft/releases/tag/0.59.2). From a224a0bf9178541bb6806c3a2057c86e8cc86a10 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 4 Oct 2017 10:14:09 +0000 Subject: [PATCH 163/177] Unify all pending changes in 4.0.0 version --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3d500703..d5287872 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,10 +6,10 @@ Released 2017-mm-dd Backward incompatible changes: - Removes `list` dataview type. -## 3.13.1 -Released 2017-10-03 +Announcements: - Upgrades yarn minimum version requirement to v0.27.5 + ## 3.13.0 Released 2017-10-02 - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. From 5d6156a257c3252d8dfc13866be33070f7b659d3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 4 Oct 2017 10:16:26 +0000 Subject: [PATCH 164/177] Add missing upgraded dep --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index e72f5676..bffdb5ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,7 @@ Backward incompatible changes: - Removes `list` dataview type. Announcements: + - Upgrades body-parser to 1.18.2. - Upgrades express to 4.16.0. - Upgrades debug to 3.1.0. - Upgrades request to 2.83.0. From 2afb6b5ac213d16b427882268afa6a647f44cc4d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 4 Oct 2017 10:32:36 +0000 Subject: [PATCH 165/177] Regenerate yarn.lock to pickup grainstore version --- yarn.lock | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/yarn.lock b/yarn.lock index 240bd4ab..6cae1a4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -444,18 +444,12 @@ debug@^1.0.4: dependencies: ms "2.0.0" -debug@^3.1.0: +debug@^3.1.0, debug@~3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: ms "2.0.0" -debug@~2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" - dependencies: - ms "0.7.1" - decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -837,11 +831,11 @@ graceful-fs@^4.1.2: resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" grainstore@~1.6.0: - version "1.6.3" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.3.tgz#6900cc811aadc1ed2c00fcd429c672f8b8e1a5cb" + version "1.6.4" + resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.4.tgz#617b93c5e2de8f544375202da89b9208a8b3d762" dependencies: carto "0.16.3" - debug "~2.2.0" + debug "~3.1.0" generic-pool "~2.2.0" millstone "0.6.17" postcss "~5.2.8" @@ -1377,10 +1371,6 @@ moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" -ms@0.7.1: - version "0.7.1" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" - ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" From 4afa7f70d7f01266a284945d298ea64fa15e061b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 4 Oct 2017 17:06:58 +0200 Subject: [PATCH 166/177] release 4.0.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index bffdb5ac..ac40dab9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 4.0.0 -Released 2017-mm-dd +Released 2017-10-04 Backward incompatible changes: - Removes `list` dataview type. From 16e82027828ca6280008bbbf9ea699c5851e383c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 4 Oct 2017 17:11:17 +0200 Subject: [PATCH 167/177] stubs next version --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ac40dab9..9b18e6cb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 4.0.1 +Released 2017-mm-dd + ## 4.0.0 Released 2017-10-04 diff --git a/package.json b/package.json index 3a1fcf7d..530a4a1f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "4.0.0", + "version": "4.0.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From c8d2c9ea37ec464bbb0a0aa7a3cb1211b78cebde Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 12:41:50 +0000 Subject: [PATCH 168/177] Do NOT throw error when not being in step context --- test/support/test-client.js | 145 ++++++++++++++++-------------------- 1 file changed, 64 insertions(+), 81 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 21918665..3e9c26f3 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -406,6 +406,8 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { var next = this; layergroupId = _layergroupId; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; var urlParams = { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 @@ -444,12 +446,6 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { if (err) { return callback(err); } - - if (layergroupId) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - } - return callback(null, dataview); } ); @@ -652,8 +648,9 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { function getTileResult(err, _layergroupId) { assert.ifError(err); - var next = this; layergroupId = _layergroupId; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; url = '/api/v1/map/' + layergroupId + '/'; @@ -727,35 +724,31 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; } - assert.response(self.server, request, expectedResponse, function(res, err) { - assert.ifError(err); - - var body; - switch (res.headers['content-type']) { - case 'image/png': - body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - break; - case 'application/x-protobuf': - body = new mapnik.VectorTile(z, x, y); - body.setDataSync(new Buffer(res.body, 'binary')); - break; - case 'application/json; charset=utf-8': - body = JSON.parse(res.body); - break; - default: - body = res.body - break; - } - - next(null, res, body); - }); + assert.response(self.server, request, expectedResponse, this); }, - function finish(err, res, image) { - if (layergroupId) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + function finish(res, err) { + if (err) { + return callback(err); } - return callback(err, res, image); + + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/x-protobuf': + body = new mapnik.VectorTile(z, x, y); + body.setDataSync(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; + } + + return callback(err, res, body); } ); }; @@ -861,6 +854,8 @@ TestClient.prototype.getStaticCenter = function (params, callback) { var next = this; layergroupid = _layergroupid; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}` @@ -884,31 +879,27 @@ TestClient.prototype.getStaticCenter = function (params, callback) { } }, params.response); - assert.response(self.server, request, expectedResponse, function(res, err) { - assert.ifError(err); - - var body; - switch (res.headers['content-type']) { - case 'image/png': - body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); - break; - case 'application/json; charset=utf-8': - body = JSON.parse(res.body); - break; - default: - body = res.body - break; - } - - next(null, res, body); - }); + assert.response(self.server, request, expectedResponse, this); }, - function finish(err, res, image) { - if (layergroupid) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + function(res, err) { + if (err) { + return callback(err); } - return callback(err, res, image); + + var body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body + break; + } + + return callback(err, res, body); } ); }; @@ -964,8 +955,9 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { function getNodeStatusResult(err, _layergroupId) { assert.ifError(err); - var next = this; layergroupId = _layergroupId; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; url = urlParser.parse(nodes[nodeName]).path; @@ -988,15 +980,13 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { } }; - assert.response(self.server, request, expectedResponse, function(res, err) { - assert.ifError(err); - next(null, res, JSON.parse(res.body)); - }); + assert.response(self.server, request, expectedResponse, this); }, - function finish(err, res, image) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - return callback(err, res, image); + function finish(res, err) { + if (err) { + return callback(err); + } + return callback(null, res, JSON.parse(res.body)); } ); }; @@ -1061,9 +1051,9 @@ TestClient.prototype.getAttributes = function(params, callback) { function getAttributes(err, _layergroupid) { assert.ifError(err); - var next = this; - layergroupid = _layergroupid; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`; @@ -1086,21 +1076,14 @@ TestClient.prototype.getAttributes = function(params, callback) { } }; - assert.response(self.server, request, expectedResponse, function (res, err) { - assert.ifError(err); - - var attributes = JSON.parse(res.body); - - next(null, res, attributes); - }); + assert.response(self.server, request, expectedResponse, this); }, - function finish(err, res, attributes) { - if (layergroupid) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + function finish(res, err) { + if (err) { + return callback(err); } - - return callback(err, res, attributes); + var attributes = JSON.parse(res.body); + return callback(null, res, attributes); } ); }; From f350206990e04373066b6aa3492d576daa2de34b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 12:54:37 +0000 Subject: [PATCH 169/177] Strict check --- test/support/assert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/assert.js b/test/support/assert.js index c5e22e93..76380833 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -137,7 +137,7 @@ assert.response = function(server, req, res, callback) { // Assert response status if (typeof status === 'number') { - if (response.statusCode != status) { + if (response.statusCode !== status) { return callback(response, new Error(colorize( '[red]{Invalid response status code.}\n' + ' Expected: [green]{' + status + '}\n' + From 6c0e6210d6ca4ccd7d8b03332721584dd1cdc97a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:15:16 +0200 Subject: [PATCH 170/177] Split response validation --- test/support/assert.js | 106 +++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/test/support/assert.js b/test/support/assert.js index 76380833..fc56a839 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -106,7 +106,6 @@ assert.response = function(server, req, res, callback) { // jshint maxcomplexity:9 function onServerListening() { - var status = res.status || res.statusCode; var requestParams = { url: 'http://' + host + ':' + port + req.url, method: req.method || 'GET', @@ -122,57 +121,70 @@ assert.response = function(server, req, res, callback) { request(requestParams, function assert$response$requestHandler(error, response, body) { listener.close(function() { response.body = response.body || body; - - // Assert response body - if (res.body) { - var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body; - if (!eql) { - return callback(response, new Error(colorize( - '[red]{Invalid response body.}\n' + - ' Expected: [green]{' + res.body + '}\n' + - ' Got: [red]{' + response.body + '}')) - ); - } - } - - // Assert response status - if (typeof status === 'number') { - if (response.statusCode !== status) { - return callback(response, new Error(colorize( - '[red]{Invalid response status code.}\n' + - ' Expected: [green]{' + status + '}\n' + - ' Got: [red]{' + response.statusCode + '}\n' + - ' Body: ' + response.body)) - ); - } - } - - // Assert response headers - if (res.headers) { - var keys = Object.keys(res.headers); - for (var i = 0, len = keys.length; i < len; ++i) { - var name = keys[i], - actual = response.headers[name.toLowerCase()], - expected = res.headers[name], - headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual; - if (!headerEql) { - return callback(response, new Error(colorize( - 'Invalid response header [bold]{' + name + '}.\n' + - ' Expected: [green]{' + expected + '}\n' + - ' Got: [red]{' + actual + '}')) - ); - } - } - } - - // Callback - callback(response); + var err = validateResponse(response, res); + return callback(response, err); }); }); } }; -// jshint maxcomplexity:6 + +function validateResponseBody(response, expected) { + if (expected.body) { + var eql = expected.body instanceof RegExp ? + expected.body.test(response.body) : + expected.body === response.body; + if (!eql) { + return new Error(colorize( + '[red]{Invalid response body.}\n' + + ' Expected: [green]{' + expected.body + '}\n' + + ' Got: [red]{' + response.body + '}') + ); + } + } +} + +function validateResponseStatus(response, expected) { + var status = expected.status || expected.statusCode; + // Assert response status + if (typeof status === 'number') { + if (response.statusCode !== status) { + return new Error(colorize( + '[red]{Invalid response status code.}\n' + + ' Expected: [green]{' + status + '}\n' + + ' Got: [red]{' + response.statusCode + '}\n' + + ' Body: ' + response.body) + ); + } + } +} + +function validateResponseHeaders(response, expected) { + // Assert response headers + if (expected.headers) { + var keys = Object.keys(expected.headers); + for (var i = 0, len = keys.length; i < len; ++i) { + var name = keys[i], + actual = response.headers[name.toLowerCase()], + expectedHeader = expected.headers[name], + headerEql = expectedHeader instanceof RegExp ? expectedHeader.test(actual) : expectedHeader === actual; + if (!headerEql) { + return new Error(colorize( + 'Invalid response header [bold]{' + name + '}.\n' + + ' Expected: [green]{' + expectedHeader + '}\n' + + ' Got: [red]{' + actual + '}') + ); + } + } + } +} + +function validateResponse(response, expected) { + // Assert response body + return validateResponseBody(response, expected) || + validateResponseStatus(response, expected) || + validateResponseHeaders(response, expected); +} // @param tolerance number of tolerated grid cell differences // jshint maxcomplexity:9 From 38c50e0bec4ba711df0f3f5c287037038329f6e7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:15:33 +0200 Subject: [PATCH 171/177] Fix jshint hint --- test/support/assert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/assert.js b/test/support/assert.js index fc56a839..44309d89 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -187,8 +187,8 @@ function validateResponse(response, expected) { } // @param tolerance number of tolerated grid cell differences -// jshint maxcomplexity:9 assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) { + // jshint maxcomplexity:9 fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8')); From 664892bba9d01441eaa16e7be4d8f859c36591d0 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:15:43 +0200 Subject: [PATCH 172/177] Complexity already fixed --- test/support/assert.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/support/assert.js b/test/support/assert.js index 44309d89..bcb186aa 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -77,7 +77,6 @@ function randomImagePath() { return path.resolve('test/results/png/image-test-' + Date.now() + '.png'); } -// jshint maxcomplexity:9 assert.response = function(server, req, res, callback) { if (!callback) { callback = res; From 5d750f3b9858921ae596b7c490fd58c7db273d94 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:24:58 +0200 Subject: [PATCH 173/177] Several jshint fixes --- test/support/test-client.js | 38 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 3e9c26f3..832d8e44 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -22,7 +22,7 @@ const MAPNIK_SUPPORTED_FORMATS = { 'grid.json': true, 'geojson': true, 'mvt': true -} +}; function TestClient(config, apiKey) { this.mapConfig = isMapConfig(config) ? config : null; @@ -115,7 +115,7 @@ module.exports.CARTOCSS = { module.exports.SQL = { EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator', ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' -} +}; TestClient.prototype.getWidget = function(widgetName, params, callback) { var self = this; @@ -479,7 +479,6 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, } }; - var layergroupId; step( function createLayergroup() { var next = this; @@ -568,7 +567,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { var layergroupId; if (params.layergroupid) { - layergroupId = params.layergroupid + layergroupId = params.layergroupid; } step( @@ -583,7 +582,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { return next(new Error('apiKey param is mandatory to create a new template')); } - params.placeholders = params.placeholders || {}; + params.placeholders = params.placeholders || {}; assert.response(self.server, { @@ -616,7 +615,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { return next(null, layergroupId); } - var data = templateId ? params.placeholders : self.mapConfig + var data = templateId ? params.placeholders : self.mapConfig; var path = templateId ? urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : url; @@ -646,6 +645,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { ); }, function getTileResult(err, _layergroupId) { + // jshint maxcomplexity:12 assert.ifError(err); layergroupId = _layergroupId; @@ -744,7 +744,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { body = JSON.parse(res.body); break; default: - body = res.body + body = res.body; break; } @@ -784,10 +784,11 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { }, expectedResponse, function(res, err) { + var parsedBody; // If there is a response, we are still interested in catching the created keys // to be able to delete them on the .drain() method. if (res) { - var parsedBody = JSON.parse(res.body); + parsedBody = JSON.parse(res.body); if (parsedBody.layergroupid) { self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; @@ -805,7 +806,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { TestClient.prototype.getStaticCenter = function (params, callback) { var self = this; - let { layergroupid, z, lat, lng, width, height, format } = params + let { layergroupid, z, lat, lng, width, height, format } = params; var url = `/api/v1/map/`; @@ -821,7 +822,7 @@ TestClient.prototype.getStaticCenter = function (params, callback) { return next(null, layergroupid); } - var data = self.mapConfig + var data = self.mapConfig; var path = url; assert.response(self.server, @@ -851,13 +852,11 @@ TestClient.prototype.getStaticCenter = function (params, callback) { function getStaticResult(err, _layergroupid) { assert.ifError(err); - var next = this; - layergroupid = _layergroupid; self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}` + url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}`; if (self.apiKey) { url += '?' + qs.stringify({api_key: self.apiKey}); @@ -895,7 +894,7 @@ TestClient.prototype.getStaticCenter = function (params, callback) { body = JSON.parse(res.body); break; default: - body = res.body + body = res.body; break; } @@ -995,11 +994,11 @@ TestClient.prototype.getAttributes = function(params, callback) { var self = this; if (!Number.isFinite(params.featureId)) { - throw new Error('featureId param must be a number') + throw new Error('featureId param must be a number'); } if (!Number.isFinite(params.layer)) { - throw new Error('layer param must be a number') + throw new Error('layer param must be a number'); } var url = '/api/v1/map'; @@ -1011,7 +1010,7 @@ TestClient.prototype.getAttributes = function(params, callback) { var layergroupid; if (params.layergroupid) { - layergroupid = params.layergroupid + layergroupid = params.layergroupid; } step( @@ -1149,12 +1148,11 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; - const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }) - const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 }) + const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }); const publicuser = global.environment.postgres.user; // we need to guarantee all new connections have the new settings - helper.cleanPGPoolConnections() + helper.cleanPGPoolConnections(); const psql = new PSQL({ user: 'postgres', From 64fe070ab26643456994c8d03ccbeff1bf4ff773 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:27:03 +0200 Subject: [PATCH 174/177] Put layergroupId handling close --- test/support/test-client.js | 43 ++++++++++++++----------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 832d8e44..3a3375cf 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -130,7 +130,6 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) }); } - var layergroupId; step( function createLayergroup() { var next = this; @@ -176,11 +175,12 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { } ); }, - function getWidgetResult(err, _layergroupId) { + function getWidgetResult(err, layergroupId) { assert.ifError(err); var next = this; - layergroupId = _layergroupId; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; var urlParams = { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 @@ -217,8 +217,6 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { ); }, function finish(err, res) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; var widget; if (!err && res.body) { widget = JSON.parse(res.body); @@ -241,7 +239,6 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) }); } - var layergroupId; step( function createLayergroup() { var next = this; @@ -287,11 +284,12 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call } ); }, - function getWidgetSearchResult(err, _layergroupId) { + function getWidgetSearchResult(err, layergroupId) { assert.ifError(err); var next = this; - layergroupId = _layergroupId; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; var urlParams = { q: userQuery, @@ -326,8 +324,6 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call ); }, function finish(err, res) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; var searchResult; if (!err && res.body) { searchResult = JSON.parse(res.body); @@ -365,7 +361,6 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { } }; - var layergroupId; step( function createLayergroup() { var next = this; @@ -401,11 +396,10 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { } ); }, - function getDataviewResult(err, _layergroupId) { + function getDataviewResult(err, layergroupId) { assert.ifError(err); var next = this; - layergroupId = _layergroupId; self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; @@ -644,11 +638,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } ); }, - function getTileResult(err, _layergroupId) { + function getTileResult(err, layergroupId) { // jshint maxcomplexity:12 assert.ifError(err); - layergroupId = _layergroupId; self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; @@ -849,14 +842,13 @@ TestClient.prototype.getStaticCenter = function (params, callback) { } ); }, - function getStaticResult(err, _layergroupid) { + function getStaticResult(err, layergroupId) { assert.ifError(err); - layergroupid = _layergroupid; - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}`; + url = `/api/v1/map/static/center/${layergroupId}/${z}/${lat}/${lng}/${width}/${height}.${format}`; if (self.apiKey) { url += '?' + qs.stringify({api_key: self.apiKey}); @@ -912,7 +904,6 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { url += '?' + qs.stringify({api_key: this.apiKey}); } - var layergroupId; var nodes = {}; step( function createLayergroup() { @@ -951,10 +942,9 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { } ); }, - function getNodeStatusResult(err, _layergroupId) { + function getNodeStatusResult(err, layergroupId) { assert.ifError(err); - layergroupId = _layergroupId; self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; @@ -1047,14 +1037,13 @@ TestClient.prototype.getAttributes = function(params, callback) { } ); }, - function getAttributes(err, _layergroupid) { + function getAttributes(err, layergroupId) { assert.ifError(err); - layergroupid = _layergroupid; - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0; + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`; + url = `/api/v1/map/${layergroupId}/${params.layer}/attributes/${params.featureId}`; if (self.apiKey) { url += '?' + qs.stringify({api_key: self.apiKey}); @@ -1126,7 +1115,7 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba // this could be removed once named maps are invalidated, otherwise you hits the cache var server = new CartodbWindshaft(serverOptions); - assert.response(self.server, requestOptions, expectedResponse, function (res, err) { + assert.response(server, requestOptions, expectedResponse, function (res, err) { helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() { return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary'))); }); From b9c511ee60b4c1e7ba5599832437b0530c797526 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:27:48 +0200 Subject: [PATCH 175/177] Remove unused file --- test/support/config.js | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 test/support/config.js diff --git a/test/support/config.js b/test/support/config.js deleted file mode 100644 index 4b8102f7..00000000 --- a/test/support/config.js +++ /dev/null @@ -1,21 +0,0 @@ -var _ = require('underscore'); - - -require(__dirname + '/test_helper'); - -module.exports = function(opts) { - - var config = { - redis_pool: { - max: 10, - idleTimeoutMillis: 1, - reapIntervalMillis: 1, - port: global.environment.redis.port - } - } - - _.extend(config, opts || {}); - - return config; -}(); - From d4015085c7fd3dde8b826bbdfc4db5218c33a9ee Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:28:01 +0200 Subject: [PATCH 176/177] Include test/support as part of jshint validation --- .jshintignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.jshintignore b/.jshintignore index d781eace..39e650b4 100644 --- a/.jshintignore +++ b/.jshintignore @@ -1,4 +1,3 @@ test/results/ test/monkey/ test/benchmark.js -test/support/ From a797e13eb30d21c851bbfd59d8ff9c5be79daee1 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 9 Oct 2017 15:51:42 +0000 Subject: [PATCH 177/177] Make all calls to finish to match (err, res) signature --- test/support/test-client.js | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 3a3375cf..3dcea842 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -117,6 +117,15 @@ module.exports.SQL = { ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator' }; +function resErr2errRes(callback) { + return (res, err) => { + if (err) { + return callback(err); + } + return callback(err, res); + }; +} + TestClient.prototype.getWidget = function(widgetName, params, callback) { var self = this; @@ -717,9 +726,9 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8'; } - assert.response(self.server, request, expectedResponse, this); + assert.response(self.server, request, expectedResponse, resErr2errRes(this)); }, - function finish(res, err) { + function finish(err, res) { if (err) { return callback(err); } @@ -870,9 +879,9 @@ TestClient.prototype.getStaticCenter = function (params, callback) { } }, params.response); - assert.response(self.server, request, expectedResponse, this); + assert.response(self.server, request, expectedResponse, resErr2errRes(this)); }, - function(res, err) { + function(err, res) { if (err) { return callback(err); } @@ -969,9 +978,9 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) { } }; - assert.response(self.server, request, expectedResponse, this); + assert.response(self.server, request, expectedResponse, resErr2errRes(this)); }, - function finish(res, err) { + function finish(err, res) { if (err) { return callback(err); } @@ -1064,9 +1073,9 @@ TestClient.prototype.getAttributes = function(params, callback) { } }; - assert.response(self.server, request, expectedResponse, this); + assert.response(self.server, request, expectedResponse, resErr2errRes(this)); }, - function finish(res, err) { + function finish(err, res) { if (err) { return callback(err); }