Merge branch 'master' into base-dataview-refactor

INSTALL.md

@@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are
 
 - Core
   - Node.js >=6.9.x
-  - yarn >=0.21.3
+  - yarn >=0.27.5 <1.0.0
   - PostgreSQL >8.3.x, PostGIS >1.5.x
   - Redis >2.4.0 (http://www.redis.io)
   - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).

NEWS.md

@@ -1,7 +1,23 @@
 # Changelog
 
-## 3.12.10
+## 3.13.1
 Released 2017-mm-dd
+ - Upgrades yarn minimum version requirement to v0.27.5
+
+Bugfixes:
+ -
+
+## 3.13.0
+Released 2017-10-02
+ - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables.
+
+Bugfixes:
+ - Bounding box parameter ignored in static named maps #735.
+ - camhaft 0.59.1 fixes duplicate columns in aggregate-intersection analysis
+
+## 3.12.10
+Released 2017-09-18
+ - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2).
 
 ## 3.12.9
 Released 2017-09-07

app.js

@@ -2,14 +2,20 @@ var http = require('http');
 var https = require('https');
 var path = require('path');
 var fs = require('fs');
-
 var _ = require('underscore');
+var semver = require('semver');
 
 // jshint undef:false
 var log = console.log.bind(console);
 var logError = console.error.bind(console);
 // jshint undef:true
 
+var nodejsVersion = process.versions.node;
+if (!semver.satisfies(nodejsVersion, '>=6.9.0')) {
+    logError(`Node version ${nodejsVersion} is not supported, please use Node.js 6.9 or higher.`);
+    process.exit(1);
+}
+
 var argv = require('yargs')
     .usage('Usage: $0 <environment> [options]')
     .help('h')

lib/cartodb/controllers/base.js

@@ -4,9 +4,6 @@ var _ = require('underscore');
 var step = require('step');
 var debug = require('debug')('windshaft:cartodb');
 
-var LZMA = require('lzma').LZMA;
-var lzmaWorker = new LZMA();
-
 // Whitelist query parameters and attach format
 var REQUEST_QUERY_PARAMS_WHITELIST = [
     'config',
@@ -28,7 +25,7 @@ function BaseController(authApi, pgConnection) {
 
 module.exports = BaseController;
 
-// jshint maxcomplexity:10
+// jshint maxcomplexity:8
 /**
  * Whitelist input and get database name & default geometry type from
  * subdomain/user metadata held in CartoDB Redis
@@ -38,35 +35,6 @@ module.exports = BaseController;
 BaseController.prototype.req2params = function(req, callback){
     var self = this;
 
-    if ( req.query.lzma ) {
-
-        // Decode (from base64)
-        var lzma = new Buffer(req.query.lzma, 'base64')
-            .toString('binary')
-            .split('')
-            .map(function(c) {
-                return c.charCodeAt(0) - 128;
-            });
-
-
-        // Decompress
-        lzmaWorker.decompress(
-            lzma,
-            function(result) {
-                req.profiler.done('lzma');
-                try {
-                    delete req.query.lzma;
-                    _.extend(req.query, JSON.parse(result));
-                    self.req2params(req, callback);
-                } catch (err) {
-                    req.profiler.done('req2params');
-                    callback(new Error('Error parsing lzma as JSON: ' + err));
-                }
-            }
-        );
-        return;
-    }
-
     var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
     if (Array.isArray(req.context.allowedQueryParams)) {
         allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams);

lib/cartodb/controllers/named_maps.js

@@ -32,7 +32,8 @@ NamedMapsController.prototype.register = function(app) {
         this.tile.bind(this));
 
     app.get(app.base_url_mapconfig +
-        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']),
+        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
+        allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
         this.staticMap.bind(this));
 };
 

lib/cartodb/middleware/lzma.js

@@ -0,0 +1,30 @@
+'use strict';
+
+var LZMA = require('lzma').LZMA;
+
+var lzmaWorker = new LZMA();
+
+module.exports = function lzmaMiddleware(req, res, next) {
+    if (!req.query.hasOwnProperty('lzma')) {
+        return next();
+    }
+
+    // Decode (from base64)
+    var lzma = new Buffer(req.query.lzma, 'base64')
+        .toString('binary')
+        .split('')
+        .map(function(c) {
+            return c.charCodeAt(0) - 128;
+        });
+
+    // Decompress
+    lzmaWorker.decompress(lzma, function(result) {
+        try {
+            delete req.query.lzma;
+            Object.assign(req.query, JSON.parse(result));
+            next();
+        } catch (err) {
+            next(new Error('Error parsing lzma as JSON: ' + err));
+        }
+    });
+};

lib/cartodb/models/dataview/aggregation.js

@@ -1,95 +1,178 @@
-var _ = require('underscore');
+const BaseDataview = require('./base');
-var BaseWidget = require('./base');
+const debug = require('debug')('windshaft:dataview:aggregation');
-var debug = require('debug')('windshaft:widget:aggregation');
 
-var dot = require('dot');
+const filteredQueryTpl = ctx => `
-dot.templateSettings.strip = false;
+    filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) _cdb_filtered_source
+        ${ctx.aggregationColumn && ctx.isFloatColumn ? `
+        WHERE
+            ${ctx.aggregationColumn} != 'infinity'::float
+        AND
+            ${ctx.aggregationColumn} != '-infinity'::float
+        AND
+            ${ctx.aggregationColumn} != 'NaN'::float` :
+        ''
+        }
+    )
+`;
 
-var filteredQueryTpl = dot.template([
+const summaryQueryTpl = ctx => `
-    'filtered_source AS (',
+    summary AS (
-    '  SELECT *',
+        SELECT
-    '  FROM ({{=it._query}}) _cdb_filtered_source',
+            count(1) AS count,
-    '  {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
+            sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count
-    '    {{=it._aggregationColumn}} != \'infinity\'::float',
+            ${ctx.isFloatColumn ? `,
-    '  AND',
+            sum(
-    '    {{=it._aggregationColumn}} != \'-infinity\'::float',
+                CASE
-    '  AND',
+                    WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float
-    '    {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
+                    THEN 1
-    ')'
+                    ELSE 0
-].join(' \n'));
+                    END
+            ) AS infinities_count,
+            sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` :
+            ''
+            }
+        FROM (${ctx.query}) _cdb_aggregation_nulls
+    )
+`;
 
-var summaryQueryTpl = dot.template([
+const rankedCategoriesQueryTpl = ctx => `
-    'summary AS (',
+    categories AS(
-    '  SELECT',
+        SELECT
-    '  count(1) AS count,',
+            ${ctx.column} AS category,
-    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
+            ${ctx.aggregationFn} AS value,
-    '  {{?it._isFloatColumn}},sum(',
+            row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
-    '    CASE',
+        FROM filtered_source
-    '      WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
+        ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
-    '      THEN 1',
+        GROUP BY ${ctx.column}
-    '      ELSE 0',
+        ORDER BY 2 DESC
-    '    END',
+    )
-    '  ) AS infinities_count,',
+`;
-    '  sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
-    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
-    ')'
-].join('\n'));
 
-var rankedCategoriesQueryTpl = dot.template([
+const categoriesSummaryMinMaxQueryTpl = () => `
-    'categories AS(',
+    categories_summary_min_max AS(
-    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
+        SELECT
-    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
+            max(value) max_val,
-    '  FROM filtered_source',
+            min(value) min_val
-    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
+        FROM categories
-    '  GROUP BY {{=it._column}}',
+    )
-    '  ORDER BY 2 DESC',
+`;
-    ')'
-].join('\n'));
 
-var categoriesSummaryMinMaxQueryTpl = dot.template([
+const categoriesSummaryCountQueryTpl = ctx => `
-    'categories_summary_min_max AS(',
+    categories_summary_count AS(
-    '  SELECT max(value) max_val, min(value) min_val',
+        SELECT count(1) AS categories_count
-    '  FROM categories',
+        FROM (
-    ')'
+            SELECT ${ctx.column} AS category
-].join('\n'));
+            FROM (${ctx.query}) _cdb_categories
+            GROUP BY ${ctx.column}
+        ) _cdb_categories_count
+    )
+`;
 
-var categoriesSummaryCountQueryTpl = dot.template([
+const specialNumericValuesColumns = () => `, nans_count, infinities_count`;
-    'categories_summary_count AS(',
-    '  SELECT count(1) AS categories_count',
-    '  FROM (',
-    '    SELECT {{=it._column}} AS category',
-    '    FROM ({{=it._query}}) _cdb_categories',
-    '    GROUP BY {{=it._column}}',
-    '  ) _cdb_categories_count',
-    ')'
-].join('\n'));
 
-var rankedAggregationQueryTpl = dot.template([
+const rankedAggregationQueryTpl = ctx => `
-    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
+    SELECT
-    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+        CAST(category AS text),
-    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
+        value,
-    '  WHERE rank < {{=it._limit}}',
+        false as agg,
-    'UNION ALL',
+        nulls_count,
-    'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,',
+        min_val,
-    '       min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+        max_val,
-    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
+        count,
-    '  WHERE rank >= {{=it._limit}}',
+        categories_count
-    'GROUP BY nulls_count, min_val, max_val, count,',
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
-    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
-].join('\n'));
+        WHERE rank < ${ctx.limit}
+    UNION ALL
+    SELECT
+        'Other' category,
+        ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value,
+        true as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank >= ${ctx.limit}
+    GROUP BY
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+`;
 
-var aggregationQueryTpl = dot.template([
+const aggregationQueryTpl = ctx => `
-    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
+    SELECT
-    '  nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+        CAST(${ctx.column} AS text) AS category,
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
+        ${ctx.aggregationFn} AS value,
-    'GROUP BY category, nulls_count, min_val, max_val, count,',
+        false as agg,
-    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+        nulls_count,
-    'ORDER BY value DESC'
+        min_val,
-].join('\n'));
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count
+    GROUP BY
+        category,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    ORDER BY value DESC
+`;
 
-var CATEGORIES_LIMIT = 6;
+const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`;
 
-var VALID_OPERATIONS = {
+const aggregationDataviewQueryTpl = ctx => `
+    WITH
+    ${filteredQueryTpl(ctx)},
+    ${summaryQueryTpl(ctx)},
+    ${rankedCategoriesQueryTpl(ctx)},
+    ${categoriesSummaryMinMaxQueryTpl(ctx)},
+    ${categoriesSummaryCountQueryTpl(ctx)}
+    ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`}
+`;
+
+const filterCategoriesQueryTpl = ctx => `
+    SELECT
+        ${ctx.column} AS category,
+        ${ctx.value} AS value
+    FROM (${ctx.query}) _cdb_aggregation_search
+    WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery}
+    GROUP BY ${ctx.column}
+`;
+
+const searchQueryTpl = ctx => `
+    WITH
+    search_unfiltered AS (
+        ${ctx.searchUnfiltered}
+    ),
+    search_filtered AS (
+        ${ctx.searchFiltered}
+    ),
+    search_union AS (
+        SELECT * FROM search_unfiltered
+        UNION ALL
+        SELECT * FROM search_filtered
+    )
+    SELECT category, sum(value) AS value
+    FROM search_union
+    GROUP BY category
+    ORDER BY value desc
+`;
+
+const CATEGORIES_LIMIT = 6;
+
+const VALID_OPERATIONS = {
     count: [],
     sum: ['aggregationColumn'],
     avg: ['aggregationColumn'],
@@ -97,7 +180,7 @@ var VALID_OPERATIONS = {
     max: ['aggregationColumn']
 };
 
-var TYPE = 'aggregation';
+const TYPE = 'aggregation';
 
 /**
  {
@@ -108,256 +191,150 @@ var TYPE = 'aggregation';
      }
  }
  */
-function Aggregation(query, options, queries) {
+module.exports = class Aggregation extends BaseDataview {
-    if (!_.isString(options.column)) {
+    constructor (query, options = {}, queries = {}) {
-        throw new Error('Aggregation expects `column` in widget options');
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.aggregation = options.aggregation;
+        this.aggregationColumn = options.aggregationColumn;
+        this._isFloatColumn = null;
     }
 
-    if (!_.isString(options.aggregation)) {
+    _checkOptions (options) {
-        throw new Error('Aggregation expects `aggregation` operation in widget options');
+        if (typeof options.column !== 'string') {
-    }
+            throw new Error(`Aggregation expects 'column' in dataview options`);
-
-    if (!VALID_OPERATIONS[options.aggregation]) {
-        throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
-    }
-
-    var requiredOptions = VALID_OPERATIONS[options.aggregation];
-    var missingOptions = _.difference(requiredOptions, Object.keys(options));
-    if (missingOptions.length > 0) {
-        throw new Error(
-            "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
-        );
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.queries = queries;
-    this.column = options.column;
-    this.aggregation = options.aggregation;
-    this.aggregationColumn = options.aggregationColumn;
-    this._isFloatColumn = null;
-}
-
-Aggregation.prototype = new BaseWidget();
-Aggregation.prototype.constructor = Aggregation;
-
-module.exports = Aggregation;
-
-Aggregation.prototype.sql = function(psql, override, callback) {
-    var self = this;
-
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    if (this.aggregationColumn && this._isFloatColumn === null) {
-        this._isFloatColumn = false;
-        this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
-            if (!err && !!type) {
-                self._isFloatColumn = type.float;
-            }
-            self.sql(psql, override, callback);
-        });
-        return null;
-    }
-
-    var _query = this.query;
-
-    var aggregationSql;
-
-    if (!!override.ownFilter) {
-        aggregationSql = [
-            this.getCategoriesCTESql(
-                _query,
-                this.column,
-                this.aggregation,
-                this.aggregationColumn,
-                this._isFloatColumn
-            ),
-            aggregationQueryTpl({
-                _isFloatColumn: this._isFloatColumn,
-                _query: _query,
-                _column: this.column,
-                _aggregation: this.getAggregationSql(),
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    } else {
-        aggregationSql = [
-            this.getCategoriesCTESql(
-                _query,
-                this.column,
-                this.aggregation,
-                this.aggregationColumn,
-                this._isFloatColumn
-            ),
-            rankedAggregationQueryTpl({
-                _isFloatColumn: this._isFloatColumn,
-                _query: _query,
-                _column: this.column,
-                _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    }
-
-    debug(aggregationSql);
-
-    return callback(null, aggregationSql);
-};
-
-Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) {
-    return [
-        "WITH",
-        [
-            filteredQueryTpl({
-                _isFloatColumn: isFloatColumn,
-                _query: this.query,
-                _column: this.column,
-                _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
-            }),
-            summaryQueryTpl({
-                _isFloatColumn: isFloatColumn,
-                _query: query,
-                _column: column,
-                _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
-            }),
-            rankedCategoriesQueryTpl({
-                _query: query,
-                _column: column,
-                _aggregation: this.getAggregationSql(),
-                _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
-            }),
-            categoriesSummaryMinMaxQueryTpl({
-                _query: query,
-                _column: column
-            }),
-            categoriesSummaryCountQueryTpl({
-                _query: query,
-                _column: column
-            })
-        ].join(',\n')
-    ].join('\n');
-};
-
-var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
-Aggregation.prototype.getAggregationSql = function() {
-    return aggregationFnQueryTpl({
-        _aggregationFn: this.aggregation,
-        _aggregationColumn: this.aggregationColumn || 1
-    });
-};
-
-Aggregation.prototype.format = function(result) {
-    var categories = [];
-    var count = 0;
-    var nulls = 0;
-    var nans = 0;
-    var infinities = 0;
-    var minValue = 0;
-    var maxValue = 0;
-    var categoriesCount = 0;
-
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        count = firstRow.count;
-        nulls = firstRow.nulls_count;
-        nans = firstRow.nans_count;
-        infinities = firstRow.infinities_count;
-        minValue = firstRow.min_val;
-        maxValue = firstRow.max_val;
-        categoriesCount = firstRow.categories_count;
-
-        result.rows.forEach(function(row) {
-            categories.push(_.omit(row, 'count', 'nulls_count', 'min_val',
-                'max_val', 'categories_count', 'nans_count', 'infinities_count'));
-        });
-    }
-
-    return {
-        aggregation: this.aggregation,
-        count: count,
-        nulls: nulls,
-        nans: nans,
-        infinities: infinities,
-        min: minValue,
-        max: maxValue,
-        categoriesCount: categoriesCount,
-        categories: categories
-    };
-};
-
-var filterCategoriesQueryTpl = dot.template([
-    'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
-    'FROM ({{=it._query}}) _cdb_aggregation_search',
-    'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
-    'GROUP BY {{=it._column}}'
-].join('\n'));
-
-var searchQueryTpl = dot.template([
-    'WITH',
-    'search_unfiltered AS (',
-    '  {{=it._searchUnfiltered}}',
-    '),',
-    'search_filtered AS (',
-    '  {{=it._searchFiltered}}',
-    '),',
-    'search_union AS (',
-    '  SELECT * FROM search_unfiltered',
-    '  UNION ALL',
-    '  SELECT * FROM search_filtered',
-    ')',
-    'SELECT category, sum(value) AS value',
-    'FROM search_union',
-    'GROUP BY category',
-    'ORDER BY value desc'
-].join('\n'));
-
-
-Aggregation.prototype.search = function(psql, userQuery, callback) {
-    var self = this;
-
-    var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
-    var _value = this.aggregation !== 'count' && this.aggregationColumn ?
-        this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)';
-
-    // TODO unfiltered will be wrong as filters are already applied at this point
-    var query = searchQueryTpl({
-        _searchUnfiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: '0',
-            _userQuery: _userQuery
-        }),
-        _searchFiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: _value,
-            _userQuery: _userQuery
-        })
-    });
-
-    psql.query(query, function(err, result) {
-        if (err) {
-            return callback(err, result);
         }
 
-        return callback(null, {type: self.getType(), categories: result.rows });
+        if (typeof options.aggregation !== 'string') {
-    }, true); // use read-only transaction
+            throw new Error(`Aggregation expects 'aggregation' operation in dataview options`);
-};
+        }
 
-Aggregation.prototype.getType = function() {
+        if (!VALID_OPERATIONS[options.aggregation]) {
-    return TYPE;
+            throw new Error(`Aggregation does not support '${options.aggregation}' operation`);
-};
+        }
 
-Aggregation.prototype.toString = function() {
+        const requiredOptions = VALID_OPERATIONS[options.aggregation];
-    return JSON.stringify({
+        const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption));
-        _type: TYPE,
+
-        _query: this.query,
+        if (missingOptions.length > 0) {
-        _column: this.column,
+            throw new Error(
-        _aggregation: this.aggregation
+                `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`
-    });
+            );
+        }
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._shouldCheckColumnType()) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const aggregationSql = aggregationDataviewQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this.aggregation,
+            aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null,
+            aggregationFn: aggregationFnQueryTpl({
+                aggregation: this.aggregation,
+                aggregationColumn: this.aggregationColumn || 1
+            }),
+            isFloatColumn: this._isFloatColumn,
+            limit: CATEGORIES_LIMIT
+        });
+
+        debug(aggregationSql);
+
+        return callback(null, aggregationSql);
+    }
+
+    _shouldCheckColumnType () {
+        return this.aggregationColumn && this._isFloatColumn === null;
+    }
+
+    format (result) {
+        const {
+            count = 0,
+            nulls_count = 0,
+            nans_count = 0,
+            infinities_count = 0,
+            min_val = 0,
+            max_val = 0,
+            categories_count = 0
+         } = result.rows[0] || {};
+
+        return {
+            aggregation: this.aggregation,
+            count: count,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count,
+            min: min_val,
+            max: max_val,
+            categoriesCount: categories_count,
+            categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg }))
+        };
+    }
+
+    search (psql, userQuery, callback) {
+        const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`);
+        const value = this.aggregation !== 'count' && this.aggregationColumn ?
+            `${this.aggregation}(${this.aggregationColumn})` :
+            'count(1)';
+
+        // TODO unfiltered will be wrong as filters are already applied at this point
+        const query = searchQueryTpl({
+            searchUnfiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: '0',
+                userQuery: escapedUserQuery
+            }),
+            searchFiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: value,
+                userQuery: escapedUserQuery
+            })
+        });
+
+        debug(query);
+
+        psql.query(query, (err, result) => {
+            if (err) {
+                return callback(err, result);
+            }
+
+            return callback(null, {type: this.getType(), categories: result.rows });
+        }, true); // use read-only transaction
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _aggregation: this.aggregation
+        });
+    }
 };

lib/cartodb/models/dataview/formula.js

@@ -1,28 +1,36 @@
-var _ = require('underscore');
+const BaseDataview = require('./base');
-var BaseWidget = require('./base');
+const debug = require('debug')('windshaft:dataview:formula');
-var debug = require('debug')('windshaft:widget:formula');
 
-var dot = require('dot');
+const countInfinitiesQueryTpl = ctx => `
-dot.templateSettings.strip = false;
+    SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities
+    WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float
+`;
 
-var formulaQueryTpl = dot.template([
+const countNansQueryTpl = ctx => `
-    'SELECT',
+    SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans
-    '  {{=it._operation}}({{=it._column}}) AS result,',
+    WHERE ${ctx.column} = 'NaN'::float
-    '  (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+`;
-    '  {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
-    '      WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
-    '  ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
-    '      WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-    'FROM ({{=it._query}}) _cdb_formula',
-    '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE',
-    '  {{=it._column}} != \'infinity\'::float',
-    'AND',
-    '  {{=it._column}} != \'-infinity\'::float',
-    'AND',
-    '  {{=it._column}} != \'NaN\'::float{{?}}'
-].join('\n'));
 
-var VALID_OPERATIONS = {
+const filterOutSpecialNumericValuesTpl = ctx => `
+    WHERE
+        ${ctx.column} != 'infinity'::float
+    AND
+        ${ctx.column} != '-infinity'::float
+    AND
+        ${ctx.column} != 'NaN'::float
+`;
+
+const formulaQueryTpl = ctx => `
+    SELECT
+        ${ctx.operation}(${ctx.column}) AS result,
+        (SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count
+        ${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''}
+        ${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''}
+    FROM (${ctx.query}) __cdb_formula
+    ${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''}
+`;
+
+const VALID_OPERATIONS = {
     count: true,
     avg: true,
     sum: true,
@@ -30,7 +38,7 @@ var VALID_OPERATIONS = {
     max: true
 };
 
-var TYPE = 'formula';
+const TYPE = 'formula';
 
 /**
  {
@@ -41,93 +49,90 @@ var TYPE = 'formula';
      }
  }
  */
-function Formula(query, options, queries) {
+module.exports = class Formula extends BaseDataview {
-    if (!_.isString(options.operation)) {
+    constructor (query, options = {}, queries = {}) {
-        throw new Error('Formula expects `operation` in widget options');
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column || '1';
+        this.operation = options.operation;
+        this._isFloatColumn = null;
     }
 
-    if (!VALID_OPERATIONS[options.operation]) {
+    _checkOptions (options) {
-        throw new Error("Formula does not support '" + options.operation + "' operation");
+        if (typeof options.operation !== 'string') {
+            throw new Error(`Formula expects 'operation' in dataview options`);
+        }
+
+        if (!VALID_OPERATIONS[options.operation]) {
+            throw new Error(`Formula does not support '${options.operation}' operation`);
+        }
+
+        if (options.operation !== 'count' && typeof options.column !== 'string') {
+            throw new Error(`Formula expects 'column' in dataview options`);
+        }
     }
 
-    if (options.operation !== 'count' && !_.isString(options.column)) {
-        throw new Error('Formula expects `column` in widget options');
-    }
 
-    BaseWidget.apply(this);
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
 
-    this.query = query;
+        if (this._isFloatColumn === null) {
-    this.queries = queries;
+            this._isFloatColumn = false;
-    this.column = options.column || '1';
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
-    this.operation = options.operation;
+                if (!err && !!type) {
-    this._isFloatColumn = null;
+                    this._isFloatColumn = type.float;
-}
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
 
-Formula.prototype = new BaseWidget();
+        const formulaSql = formulaQueryTpl({
-Formula.prototype.constructor = Formula;
+            isFloatColumn: this._isFloatColumn,
-
+            query: this.query,
-module.exports = Formula;
+            operation: this.operation,
-
+            column: this.column
-Formula.prototype.sql = function(psql, override, callback) {
-    var self = this;
-
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    if (this._isFloatColumn === null) {
-        this._isFloatColumn = false;
-        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
-            if (!err && !!type) {
-                self._isFloatColumn = type.float;
-            }
-            self.sql(psql, override, callback);
         });
-        return null;
+
+        debug(formulaSql);
+
+        return callback(null, formulaSql);
     }
 
-    var formulaSql = formulaQueryTpl({
+    format (res) {
-        _isFloatColumn: this._isFloatColumn,
+        const {
-        _query: this.query,
+            result = 0,
-        _operation: this.operation,
+            nulls_count = 0,
-        _column: this.column
+            nans_count,
-    });
+            infinities_count
+        } = res.rows[0] || {};
 
-    debug(formulaSql);
+        return {
-
+            operation: this.operation,
-    return callback(null, formulaSql);
+            result,
-};
+            nulls: nulls_count,
-
+            nans: nans_count,
-Formula.prototype.format = function(result) {
+            infinities: infinities_count
-    var formattedResult = {
+        };
-        operation: this.operation,
-        result: 0,
-        nulls: 0,
-        nans: 0,
-        infinities: 0
-    };
-
-    if (result.rows.length) {
-        formattedResult.operation = this.operation;
-        formattedResult.result = result.rows[0].result;
-        formattedResult.nulls = result.rows[0].nulls_count;
-        formattedResult.nans = result.rows[0].nans_count;
-        formattedResult.infinities = result.rows[0].infinities_count;
     }
 
-    return formattedResult;
+    getType () {
-};
+        return TYPE;
+    }
 
-Formula.prototype.getType = function() {
+    toString () {
-    return TYPE;
+        return JSON.stringify({
-};
+            _type: TYPE,
-
+            _query: this.query,
-Formula.prototype.toString = function() {
+            _column: this.column,
-    return JSON.stringify({
+            _operation: this.operation
-        _type: TYPE,
+        });
-        _query: this.query,
+    }
-        _column: this.column,
-        _operation: this.operation
-    });
 };

lib/cartodb/models/dataview/histogram.js

@@ -1,717 +1,72 @@
-var _ = require('underscore');
+const debug = require('debug')('windshaft:dataview:histogram');
-var BaseWidget = require('./base');
+const NumericHistogram = require('./histograms/numeric-histogram');
-var debug = require('debug')('windshaft:dataview:histogram');
+const DateHistogram = require('./histograms/date-histogram');
 
-var dot = require('dot');
+const DATE_HISTOGRAM = 'DateHistogram';
-dot.templateSettings.strip = false;
+const NUMERIC_HISTOGRAM = 'NumericHistogram';
 
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+module.exports = class Histogram {
+    constructor (query, options, queries) {
+        this.query = query;
+        this.options = options || {};
+        this.queries = queries;
 
-var dateIntervalQueryTpl = dot.template([
+        this.histogramImplementation = this._getHistogramImplementation();
-    'WITH',
-    '__cdb_dates AS (',
-    '    SELECT',
-    '        MAX({{=it.column}}::timestamp) AS __cdb_end,',
-    '        MIN({{=it.column}}::timestamp) AS __cdb_start',
-    '    FROM ({{=it.query}}) __cdb_source',
-    '),',
-    '__cdb_interval_in_days AS (',
-    '    SELECT' ,
-    '        DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days',
-    '    FROM __cdb_dates',
-    '),',
-    '__cdb_interval_in_hours AS (',
-    '    SELECT',
-    '        __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours',
-    '    FROM __cdb_interval_in_days, __cdb_dates',
-    '),',
-    '__cdb_interval_in_minutes AS (',
-    '    SELECT',
-    '        __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes',
-    '    FROM __cdb_interval_in_hours, __cdb_dates',
-    '),',
-    '__cdb_interval_in_seconds AS (',
-    '    SELECT',
-    '        __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds',
-    '    FROM __cdb_interval_in_minutes, __cdb_dates',
-    ')',
-    'SELECT',
-    '    ROUND(__cdb_days / 365) AS year,',
-    '    ROUND(__cdb_days / 90) AS quarter,',
-    '    ROUND(__cdb_days / 30) AS month,',
-    '    ROUND(__cdb_days / 7) AS week,',
-    '    __cdb_days AS day,',
-    '    __cdb_hours AS hour,',
-    '    __cdb_minutes AS minute,',
-    '    __cdb_seconds AS second',
-    'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds'
-].join('\n'));
-
-var MAX_INTERVAL_VALUE = 366;
-var BIN_MIN_NUMBER = 6;
-var BIN_MAX_NUMBER = 48;
-
-var filteredQueryTpl = dot.template([
-    '__cdb_filtered_source AS (',
-    '  SELECT *',
-    '  FROM ({{=it._query}}) __cdb_filtered_source_query',
-    '  WHERE',
-    '    {{=it._column}} IS NOT NULL',
-    '  {{?it._isFloatColumn}}AND',
-    '    {{=it._column}} != \'infinity\'::float',
-    '  AND',
-    '    {{=it._column}} != \'-infinity\'::float',
-    '  AND',
-    '    {{=it._column}} != \'NaN\'::float{{?}}',
-    ')'
-].join(' \n'));
-
-var basicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '  SELECT',
-    '    max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,',
-    '    avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
-    '  FROM __cdb_filtered_source',
-    ')'
-].join(' \n'));
-
-var overrideBasicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '  SELECT',
-    '    max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,',
-    '    avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
-    '  FROM __cdb_filtered_source',
-    ')'
-].join('\n'));
-
-var iqrQueryTpl = dot.template([
-    '__cdb_iqrange AS (',
-    '  SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr',
-    '  FROM (',
-    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
-    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
-    '    ) AS quartile',
-    '    FROM __cdb_filtered_source) _cdb_quartiles',
-    '    WHERE quartile = 1 or quartile = 3',
-    '    GROUP BY quartile',
-    '  ) __cdb_iqr',
-    ')'
-].join('\n'));
-
-var binsQueryTpl = dot.template([
-    '__cdb_bins AS (',
-    '  SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0',
-    '      THEN 1',
-    '      ELSE GREATEST(',
-    '        LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),',
-    '        LEAST(',
-    '          CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),',
-    '          {{=it._maxBins}}',
-    '        )',
-    '      )',
-    '    END AS __cdb_bins_number',
-    '  FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source',
-    '  LIMIT 1',
-    ')'
-].join('\n'));
-
-var overrideBinsQueryTpl = dot.template([
-    '__cdb_bins AS (',
-    '  SELECT {{=it._bins}} AS __cdb_bins_number',
-    ')'
-].join('\n'));
-
-var nullsQueryTpl = dot.template([
-    '__cdb_nulls AS (',
-    '  SELECT',
-    '    count(*) AS __cdb_nulls_count',
-    '  FROM ({{=it._query}}) __cdb_histogram_nulls',
-    '  WHERE {{=it._column}} IS NULL',
-    ')'
-].join('\n'));
-
-var infinitiesQueryTpl = dot.template([
-    '__cdb_infinities AS (',
-    '  SELECT',
-    '    count(*) AS __cdb_infinities_count',
-    '  FROM ({{=it._query}}) __cdb_infinities_query',
-    '  WHERE',
-    '    {{=it._column}} = \'infinity\'::float',
-    '  OR',
-    '    {{=it._column}} = \'-infinity\'::float',
-    ')'
-].join('\n'));
-
-var nansQueryTpl = dot.template([
-    '__cdb_nans AS (',
-    '  SELECT',
-    '    count(*) AS __cdb_nans_count',
-    '  FROM ({{=it._query}}) __cdb_nans_query',
-    '  WHERE {{=it._column}} = \'NaN\'::float',
-    ')'
-].join('\n'));
-
-var histogramQueryTpl = dot.template([
-    'SELECT',
-    '    (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
-    '    __cdb_bins_number AS bins_number,',
-    '    __cdb_nulls_count AS nulls_count,',
-    '    {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,',
-    '    __cdb_nans_count AS nans_count,{{?}}',
-    '    __cdb_avg_val AS avg_val,',
-    '    CASE WHEN __cdb_min_val = __cdb_max_val',
-    '      THEN 0',
-    '      ELSE GREATEST(',
-    '           1,',
-    '           LEAST(',
-    '               WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),',
-    '               __cdb_bins_number',
-    '           )',
-    '      ) - 1',
-    '    END AS bin,',
-    '    min({{=it._column}})::numeric AS min,',
-    '    max({{=it._column}})::numeric AS max,',
-    '    avg({{=it._column}})::numeric AS avg,',
-    '    count(*) AS freq',
-    'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,',
-    '       __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}',
-    'GROUP BY bin, bins_number, bin_width, nulls_count,',
-    '  avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
-    'ORDER BY bin'
-].join('\n'));
-
-var dateBasicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '    SELECT',
-    '        max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,',
-    '        min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,',
-    '        avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
-    '        min(date_trunc(',
-    '           \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '        )) AS __cdb_start_date,',
-    '        max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,',
-    '        count(1) AS __cdb_total_rows',
-    '    FROM ({{=it._query}}) __cdb_basics_query',
-    ')'
-].join(' \n'));
-
-var dateOverrideBasicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '    SELECT',
-    '        max({{=it._end}})::float AS __cdb_max_val,',
-    '        min({{=it._start}})::float AS __cdb_min_val,',
-    '        avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
-    '        min(',
-    '           date_trunc(',
-    '               \'{{=it._aggregation}}\',',
-    '               TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '           )',
-    '        ) AS __cdb_start_date,',
-    '        max(',
-    '           TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '        ) AS __cdb_end_date,',
-    '        count(1) AS __cdb_total_rows',
-    '    FROM ({{=it._query}}) __cdb_basics_query',
-    ')'
-].join(' \n'));
-
-var dateBinsQueryTpl = dot.template([
-    '__cdb_bins AS (',
-    '    SELECT',
-    '        __cdb_bins_array,',
-    '        ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number',
-    '    FROM (',
-    '       SELECT',
-    '           ARRAY(',
-    '               SELECT GENERATE_SERIES(',
-    '                   __cdb_start_date::timestamptz,',
-    '                   __cdb_end_date::timestamptz,',
-    '                   {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval',
-    '                )',
-    '           ) AS __cdb_bins_array',
-    '       FROM __cdb_basics',
-    '    ) __cdb_bins_array_query',
-    ')'
-].join('\n'));
-
-var dateHistogramQueryTpl = dot.template([
-    'SELECT',
-    '    (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
-    '    __cdb_bins_number AS bins_number,',
-    '    __cdb_nulls_count AS nulls_count,',
-    '    CASE WHEN __cdb_min_val = __cdb_max_val',
-    '      THEN 0',
-    '      ELSE GREATEST(1, LEAST(',
-    '        WIDTH_BUCKET(',
-    '          {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',',
-    '          __cdb_bins_array',
-    '        ),',
-    '        __cdb_bins_number',
-    '      )) - 1',
-    '    END AS bin,',
-    '    min(',
-    '       date_part(',
-    '           \'epoch\', ',
-    '           date_trunc(',
-    '               \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '           ) AT TIME ZONE \'{{=it._offset}}\'',
-    '       )',
-    '    )::numeric AS timestamp,',
-    '    date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,',
-    '    min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
-    '    max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
-    '    avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
-    '    count(*) AS freq',
-    'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls',
-    'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
-    'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start',
-    'ORDER BY bin'
-].join('\n'));
-
-var TYPE = 'histogram';
-
-/**
-Numeric histogram:
-{
-    type: 'histogram',
-    options: {
-        column: 'name', // column data type: numeric
-        bins: 10 // OPTIONAL
-    }
-}
-
-Time series:
-{
-    type: 'histogram',
-    options: {
-        column: 'date', // column data type: date
-        aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric)
-        offset: -7200 // OPTIONAL (UTC offset in seconds)
-    }
- }
- */
-function Histogram(query, options, queries) {
-    if (!_.isString(options.column)) {
-        throw new Error('Histogram expects `column` in widget options');
     }
 
-    this.query = query;
+    _getHistogramImplementation (override) {
-    this.queries = queries;
+        let implementation = null;
-    this.column = options.column;
-    this.bins = options.bins;
-    this.aggregation = options.aggregation;
-    this.offset = options.offset;
 
-    this._columnType = null;
+        switch (this._getHistogramSubtype(override)) {
-}
+            case DATE_HISTOGRAM:
-
+                debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override);
-Histogram.prototype = new BaseWidget();
+                implementation = new DateHistogram(this.query, this.options, this.queries);
-Histogram.prototype.constructor = Histogram;
+                break;
-
+            case NUMERIC_HISTOGRAM:
-module.exports = Histogram;
+                debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override);
-
+                implementation = new NumericHistogram(this.query, this.options, this.queries);
-Histogram.prototype.sql = function(psql, override, callback) {
+                break;
-    var self = this;
+            default:
-
+                throw new Error('Unsupported Histogram type');
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    if (this._columnType === null) {
-        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
-            // assume numeric, will fail later
-            self._columnType = 'numeric';
-            if (!err && !!type) {
-                self._columnType = Object.keys(type).find(function (key) {
-                    return type[key];
-                });
-            }
-            self.sql(psql, override, callback);
-        }, true); // use read-only transaction
-        return null;
-    }
-
-    this._buildQuery(psql, override, callback);
-};
-
-Histogram.prototype.isDateHistogram = function (override) {
-    return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined);
-};
-
-Histogram.prototype._buildQuery = function (psql, override, callback) {
-    var filteredQuery, basicsQuery, binsQuery;
-    var _column = this.column;
-    var _query = this.query;
-
-    if (this.isDateHistogram(override)) {
-        return this._buildDateHistogramQuery(psql, override, callback);
-    }
-
-    if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
-    }
-
-    filteredQuery = filteredQueryTpl({
-        _isFloatColumn: this._columnType === 'float',
-        _query: _query,
-        _column: _column
-    });
-
-    if (this._shouldOverride(override)) {
-        debug('overriding with %j', override);
-        basicsQuery = overrideBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _start: getBinStart(override),
-            _end: getBinEnd(override)
-        });
-
-        binsQuery = [
-            overrideBinsQueryTpl({
-                _bins: override.bins
-            })
-        ].join(',\n');
-    } else {
-        basicsQuery = basicsQueryTpl({
-            _query: _query,
-            _column: _column
-        });
-
-        if (this._shouldOverrideBins(override)) {
-            binsQuery = [
-                overrideBinsQueryTpl({
-                    _bins: override.bins
-                })
-            ].join(',\n');
-        } else {
-            binsQuery = [
-                iqrQueryTpl({
-                    _query: _query,
-                    _column: _column
-                }),
-                binsQueryTpl({
-                    _query: _query,
-                    _minBins: BIN_MIN_NUMBER,
-                    _maxBins: BIN_MAX_NUMBER
-                })
-            ].join(',\n');
-        }
-    }
-
-    var cteSql = [
-        filteredQuery,
-        basicsQuery,
-        binsQuery,
-        nullsQueryTpl({
-            _query: _query,
-            _column: _column
-        })
-    ];
-
-    if (this._columnType === 'float') {
-        cteSql.push(
-            infinitiesQueryTpl({
-                _query: _query,
-                _column: _column
-            }),
-            nansQueryTpl({
-                _query: _query,
-                _column: _column
-            })
-        );
-    }
-
-    var histogramSql = [
-        "WITH",
-        cteSql.join(',\n'),
-        histogramQueryTpl({
-            _isFloatColumn: this._columnType === 'float',
-            _query: _query,
-            _column: _column
-        })
-    ].join('\n');
-
-    debug(histogramSql);
-
-    return callback(null, histogramSql);
-};
-
-Histogram.prototype._shouldOverride = function (override) {
-    return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
-};
-
-Histogram.prototype._shouldOverrideBins = function (override) {
-    return override && _.has(override, 'bins');
-};
-
-var DATE_AGGREGATIONS = {
-    'auto': true,
-    'minute': true,
-    'hour': true,
-    'day': true,
-    'week': true,
-    'month': true,
-    'quarter': true,
-    'year': true
-};
-
-Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) {
-    var _column = this.column;
-    var _query = this.query;
-    var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
-    var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset;
-
-    if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) {
-        return callback(new Error('Invalid aggregation value. Valid ones: ' +
-            Object.keys(DATE_AGGREGATIONS).join(', ')
-        ));
-    }
-
-    if (_aggregation === 'auto') {
-        this.getAutomaticAggregation(psql, function (err, aggregation) {
-            if (err || aggregation === 'none') {
-                this.aggregation = 'day';
-            } else {
-                this.aggregation = aggregation;
-            }
-            override.aggregation = this.aggregation;
-            this._buildDateHistogramQuery(psql, override, callback);
-        }.bind(this));
-        return null;
-    }
-
-    var dateBasicsQuery;
-
-    if (override && _.has(override, 'start') && _.has(override, 'end')) {
-        dateBasicsQuery = dateOverrideBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _aggregation: _aggregation,
-            _start: getBinStart(override),
-            _end: getBinEnd(override),
-            _offset: parseOffset(_offset, _aggregation)
-        });
-    } else {
-        dateBasicsQuery = dateBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _aggregation: _aggregation,
-            _offset: parseOffset(_offset, _aggregation)
-        });
-    }
-
-    var dateBinsQuery = [
-        dateBinsQueryTpl({
-            _aggregation: _aggregation
-        })
-    ].join(',\n');
-
-    var nullsQuery = nullsQueryTpl({
-        _query: _query,
-        _column: _column
-    });
-
-    var dateHistogramQuery = dateHistogramQueryTpl({
-        _query: _query,
-        _column: _column,
-        _aggregation: _aggregation,
-        _offset: parseOffset(_offset, _aggregation)
-    });
-
-    var histogramSql = [
-        "WITH",
-        [
-            dateBasicsQuery,
-            dateBinsQuery,
-            nullsQuery
-        ].join(',\n'),
-        dateHistogramQuery
-    ].join('\n');
-
-    debug(histogramSql);
-
-    return callback(null, histogramSql);
-};
-
-Histogram.prototype.getAutomaticAggregation = function (psql, callback) {
-    var dateIntervalQuery = dateIntervalQueryTpl({
-        query: this.query,
-        column: this.column
-    });
-
-    debug(dateIntervalQuery);
-
-    psql.query(dateIntervalQuery, function (err, result) {
-        if (err) {
-            return callback(err);
         }
 
-        var aggegations = result.rows[0];
+        return implementation;
-        var aggregation = Object.keys(aggegations)
+    }
-            .map(function (key) {
-                return {
-                    name: key,
-                    value: aggegations[key]
-                };
-            })
-            .reduce(function (closer, current) {
-                if (current.value > MAX_INTERVAL_VALUE) {
-                    return closer;
-                }
 
-                var closerDiff = MAX_INTERVAL_VALUE - closer.value;
+    _getHistogramSubtype (override) {
-                var currentDiff = MAX_INTERVAL_VALUE - current.value;
+        if(this._isDateHistogram(override)) {
-
+            return DATE_HISTOGRAM;
-                if (Number.isFinite(current.value) && closerDiff > currentDiff) {
-                    return current;
-                }
-
-                return closer;
-            }, { name: 'none', value: -1 });
-
-        callback(null, aggregation.name);
-    });
-};
-
-Histogram.prototype.format = function(result, override) {
-    override = override || {};
-    var buckets = [];
-
-    var binsCount = getBinsCount(override);
-    var width = getWidth(override);
-    var binsStart = getBinStart(override);
-    var nulls = 0;
-    var infinities = 0;
-    var nans = 0;
-    var avg;
-    var timestampStart;
-    var aggregation;
-    var offset;
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        binsCount = firstRow.bins_number;
-        width = firstRow.bin_width || width;
-        avg = firstRow.avg_val;
-        nulls = firstRow.nulls_count;
-        timestampStart = firstRow.timestamp_start;
-        infinities = firstRow.infinities_count;
-        nans = firstRow.nans_count;
-        binsStart = populateBinStart(override, firstRow);
-
-        if (Number.isFinite(timestampStart)) {
-            aggregation = getAggregation(override, this.aggregation);
-            offset = getOffset(override, this.offset);
         }
 
-        buckets = result.rows.map(function(row) {
+        return NUMERIC_HISTOGRAM;
-            return _.omit(
-                row,
-                'bins_number',
-                'bin_width',
-                'nulls_count',
-                'infinities_count',
-                'nans_count',
-                'avg_val',
-                'timestamp_start'
-            );
-        });
     }
 
-    return {
+    _isDateHistogram (override = {}) {
-        aggregation: aggregation,
+        return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation'));
-        offset: offset,
+    }
-        timestamp_start: timestampStart,
+
-        bin_width: width,
+    getResult (psql, override, callback) {
-        bins_count: binsCount,
+        this.histogramImplementation = this._getHistogramImplementation(override);
-        bins_start: binsStart,
+        this.histogramImplementation.getResult(psql, override, callback);
-        nulls: nulls,
+    }
-        infinities: infinities,
+
-        nans: nans,
+    // In order to keep previous behaviour with overviews,
-        avg: avg,
+    // we have to expose the following methods to bypass
-        bins: buckets
+    // the concrete overview implementation
-    };
+
-};
+    sql (psql, override, callback) {
-
+        this.histogramImplementation.sql(psql, override, callback);
-function getAggregation(override, aggregation) {
+    }
-    return override && override.aggregation ? override.aggregation : aggregation;
+
-}
+    format (result, override) {
-
+        return this.histogramImplementation.format(result, override);
-function getOffset(override, offset) {
+    }
-    if (override && override.offset) {
+
-        return override.offset;
+    getType () {
-    }
+        return this.histogramImplementation.getType();
-    if (offset) {
+    }
-        return offset;
+
-    }
+    toString () {
-
+        return this.histogramImplementation.toString();
-    return 0;
+    }
-}
-
-function getBinStart(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.min(override.start, override.end);
-    }
-    return override.start || 0;
-}
-
-function getBinEnd(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.max(override.start, override.end);
-    }
-    return override.end || 0;
-}
-
-function getBinsCount(override) {
-    return override.bins || 0;
-}
-
-function getWidth(override) {
-    var width = 0;
-    var binsCount = override.bins;
-
-    if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) {
-        width = (override.end - override.start) / binsCount;
-    }
-
-    return width;
-}
-
-function parseOffset(offset, aggregation) {
-    if (!offset) {
-        return '0';
-    }
-    if (aggregation === 'hour' || aggregation === 'minute') {
-        return '0';
-    }
-
-    var offsetInHours = Math.ceil(offset / 3600);
-    return '' + offsetInHours;
-}
-
-function populateBinStart(override, firstRow) {
-    var binStart;
-
-    if (firstRow.hasOwnProperty('timestamp')) {
-        binStart = firstRow.timestamp;
-    } else if (override.hasOwnProperty('start')) {
-        binStart = getBinStart(override);
-    } else {
-        binStart = firstRow.min;
-    }
-
-    return binStart;
-}
-
-Histogram.prototype.getType = function() {
-    return TYPE;
-};
-
-Histogram.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _column: this.column,
-        _query: this.query
-    });
 };

lib/cartodb/models/dataview/histograms/base-histogram.js

@@ -0,0 +1,85 @@
+const BaseDataview = require('../base');
+
+const TYPE = 'histogram';
+
+module.exports = class BaseHistogram extends BaseDataview {
+    constructor (query, options, queries) {
+        super();
+
+        if (typeof options.column !== 'string') {
+            throw new Error('Histogram expects `column` in widget options');
+        }
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.bins = options.bins;
+
+        this._columnType = null;
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._columnType === null) {
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                // assume numeric, will fail later
+                this._columnType = 'numeric';
+                if (!err && !!type) {
+                    this._columnType = Object.keys(type).find(function (key) {
+                        return type[key];
+                    });
+                }
+                this.sql(psql, override, callback);
+            }, true); // use read-only transaction
+            return null;
+        }
+
+        return this._buildQuery(psql, override, callback);
+    }
+
+    format (result, override) {
+        const histogram = this._getSummary(result, override);
+        histogram.bins = this._getBuckets(result);
+        return histogram;
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _column: this.column,
+            _query: this.query
+        });
+    }
+
+    _hasOverridenRange (override) {
+        return override && override.hasOwnProperty('start') && override.hasOwnProperty('end');
+    }
+
+    _getBinStart (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.min(override.start, override.end);
+        }
+
+        return override.start || 0;
+    }
+
+    _getBinEnd (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.max(override.start, override.end);
+        }
+
+        return override.end || 0;
+    }
+
+    _getBinsCount (override = {}) {
+        return override.bins || 0;
+    }
+};

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -0,0 +1,302 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:date-histogram');
+
+const dateIntervalQueryTpl = ctx => `
+    WITH
+    __cdb_dates AS (
+        SELECT
+            MAX(${ctx.column}::timestamp) AS __cdb_end,
+            MIN(${ctx.column}::timestamp) AS __cdb_start
+        FROM (${ctx.query}) __cdb_source
+    ),
+    __cdb_interval_in_days AS (
+        SELECT
+            DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days
+        FROM __cdb_dates
+    ),
+    __cdb_interval_in_hours AS (
+        SELECT
+            __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours
+        FROM __cdb_interval_in_days, __cdb_dates
+    ),
+    __cdb_interval_in_minutes AS (
+        SELECT
+            __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes
+        FROM __cdb_interval_in_hours, __cdb_dates
+    ),
+    __cdb_interval_in_seconds AS (
+        SELECT
+            __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds
+        FROM __cdb_interval_in_minutes, __cdb_dates
+    )
+    SELECT
+        ROUND(__cdb_days / 365) AS year,
+        ROUND(__cdb_days / 90) AS quarter,
+        ROUND(__cdb_days / 30) AS month,
+        ROUND(__cdb_days / 7) AS week,
+        __cdb_days AS day,
+        __cdb_hours AS hour,
+        __cdb_minutes AS minute,
+        __cdb_seconds AS second
+    FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
+`;
+
+const nullsQueryTpl = ctx => `
+    __cdb_nulls AS (
+        SELECT
+            count(*) AS __cdb_nulls_count
+        FROM (${ctx.query}) __cdb_histogram_nulls
+        WHERE ${ctx.column} IS NULL
+    )
+`;
+
+const dateBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(date_part('epoch', ${ctx.column})) AS __cdb_max_val,
+            min(date_part('epoch', ${ctx.column})) AS __cdb_min_val,
+            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
+            min(
+                date_trunc(
+                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
+                )
+            ) AS __cdb_start_date,
+            max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date,
+            count(1) AS __cdb_total_rows
+        FROM (${ctx.query}) __cdb_basics_query
+    )
+`;
+
+const dateOverrideBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.end})::float AS __cdb_max_val,
+            min(${ctx.start})::float AS __cdb_min_val,
+            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
+            min(
+                date_trunc(
+                    '${ctx.aggregation}',
+                    TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}'
+                )
+            ) AS __cdb_start_date,
+            max(
+                TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}'
+            ) AS __cdb_end_date,
+            count(1) AS __cdb_total_rows
+        FROM (${ctx.query}) __cdb_basics_query
+    )
+`;
+
+const dateBinsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT
+            __cdb_bins_array,
+            ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number
+        FROM (
+            SELECT
+                ARRAY(
+                    SELECT GENERATE_SERIES(
+                        __cdb_start_date::timestamptz,
+                        __cdb_end_date::timestamptz,
+                        ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`}
+                    )
+                ) AS __cdb_bins_array
+            FROM __cdb_basics
+        ) __cdb_bins_array_query
+    )
+`;
+
+const dateHistogramQueryTpl = ctx => `
+    SELECT
+        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
+        __cdb_bins_number AS bins_number,
+        __cdb_nulls_count AS nulls_count,
+        CASE WHEN __cdb_min_val = __cdb_max_val
+        THEN 0
+        ELSE GREATEST(
+            1,
+            LEAST(
+                WIDTH_BUCKET(
+                    ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}',
+                    __cdb_bins_array
+                ),
+                __cdb_bins_number
+            )
+        ) - 1
+        END AS bin,
+        min(
+            date_part(
+                'epoch',
+                date_trunc(
+                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
+                ) AT TIME ZONE '${ctx.offset}'
+            )
+        )::numeric AS timestamp,
+        date_part('epoch', __cdb_start_date)::numeric AS timestamp_start,
+        min(date_part('epoch', ${ctx.column}))::numeric AS min,
+        max(date_part('epoch', ${ctx.column}))::numeric AS max,
+        avg(date_part('epoch', ${ctx.column}))::numeric AS avg,
+        count(*) AS freq
+    FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls
+    WHERE date_part('epoch', ${ctx.column}) IS NOT NULL
+    GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start
+    ORDER BY bin
+`;
+
+const MAX_INTERVAL_VALUE = 366;
+
+const DATE_AGGREGATIONS = {
+    'auto': true,
+    'minute': true,
+    'hour': true,
+    'day': true,
+    'week': true,
+    'month': true,
+    'quarter': true,
+    'year': true
+};
+
+/**
+    date_histogram: {
+        type: 'histogram',
+        options: {
+            column: 'date', // column data type: date
+            aggregation: 'day' // MANDATORY
+            offset: -7200 // OPTIONAL (UTC offset in seconds)
+        }
+    }
+*/
+module.exports = class DateHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+
+        this.aggregation = options.aggregation;
+        this.offset = options.offset;
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+            WITH
+            ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)},
+            ${dateBinsQueryTpl(ctx)},
+            ${nullsQueryTpl(ctx)}
+            ${dateHistogramQueryTpl(ctx)}
+        `;
+    }
+
+    _buildQuery (psql, override, callback) {
+        if (!this._isValidAggregation(override)) {
+            return callback(new Error('Invalid aggregation value. Valid ones: ' +
+                Object.keys(DATE_AGGREGATIONS).join(', ')
+            ));
+        }
+
+        if (this._getAggregation(override) === 'auto') {
+            this._getAutomaticAggregation(psql, function (err, aggregation) {
+                if (err || aggregation === 'none') {
+                    this.aggregation = 'day';
+                } else {
+                    this.aggregation = aggregation;
+                }
+                override.aggregation = this.aggregation;
+                this._buildQuery(psql, override, callback);
+            }.bind(this));
+            return null;
+        }
+
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this._getAggregation(override),
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            offset: this._parseOffset(override)
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _isValidAggregation (override) {
+        return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override));
+    }
+
+    _getAutomaticAggregation (psql, callback) {
+        const dateIntervalQuery = dateIntervalQueryTpl({
+            query: this.query,
+            column: this.column
+        });
+
+        psql.query(dateIntervalQuery, function (err, result) {
+            if (err) {
+                return callback(err);
+            }
+
+            const aggegations = result.rows[0];
+            const aggregation = Object.keys(aggegations)
+                .map(key => ({ name: key, value: aggegations[key] }))
+                .reduce((closer, current) => {
+                    if (current.value > MAX_INTERVAL_VALUE) {
+                        return closer;
+                    }
+
+                    const closerDiff = MAX_INTERVAL_VALUE - closer.value;
+                    const currentDiff = MAX_INTERVAL_VALUE - current.value;
+
+                    if (Number.isFinite(current.value) && closerDiff > currentDiff) {
+                        return current;
+                    }
+
+                    return closer;
+                }, { name: 'none', value: -1 });
+
+            callback(null, aggregation.name);
+        });
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            aggregation: this._getAggregation(override),
+            offset: this._getOffset(override),
+            timestamp_start: firstRow.timestamp_start,
+
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: firstRow.timestamp,
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
+    }
+
+    _getAggregation (override = {}) {
+        return override.aggregation ? override.aggregation : this.aggregation;
+    }
+
+    _getOffset (override = {}) {
+        return Number.isFinite(override.offset) ? override.offset : (this.offset || 0);
+    }
+
+    _parseOffset (override) {
+        if (this._shouldIgnoreOffset(override)) {
+            return '0';
+        }
+
+        const offsetInHours = Math.ceil(this._getOffset(override) / 3600);
+
+        return '' + offsetInHours;
+    }
+
+    _shouldIgnoreOffset (override) {
+        return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute');
+    }
+};

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -0,0 +1,234 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:numeric-histogram');
+
+const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`;
+
+const filterOutSpecialNumericValues = ctx => `
+        ${ctx.column} != 'infinity'::float
+    AND
+        ${ctx.column} != '-infinity'::float
+    AND
+        ${ctx.column} != 'NaN'::float
+`;
+
+const filteredQueryTpl = ctx => `
+    __cdb_filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) __cdb_filtered_source_query
+        WHERE ${ctx.column} IS NOT NULL
+        ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''}
+    )
+`;
+
+const basicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val,
+            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+const overrideBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val,
+            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+const iqrQueryTpl = ctx => `
+    __cdb_iqrange AS (
+        SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
+        FROM (
+            SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (
+                SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column}
+            ) AS quartile
+            FROM __cdb_filtered_source) _cdb_quartiles
+            WHERE quartile = 1 or quartile = 3
+            GROUP BY quartile
+        ) __cdb_iqr
+    )
+`;
+
+const binsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT
+            CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
+            THEN 1
+            ELSE GREATEST(
+                LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)),
+                LEAST(
+                    CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),
+                    ${ctx.maxBins}
+                )
+            )
+            END AS __cdb_bins_number
+        FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source
+        LIMIT 1
+    )
+`;
+
+const overrideBinsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT ${ctx.override.bins} AS __cdb_bins_number
+    )
+`;
+
+const nullsQueryTpl = ctx => `
+    __cdb_nulls AS (
+        SELECT
+            count(*) AS __cdb_nulls_count
+        FROM (${ctx.query}) __cdb_histogram_nulls
+        WHERE ${ctx.column} IS NULL
+    )
+`;
+
+const infinitiesQueryTpl = ctx => `
+    __cdb_infinities AS (
+        SELECT
+            count(*) AS __cdb_infinities_count
+        FROM (${ctx.query}) __cdb_infinities_query
+        WHERE
+            ${ctx.column} = 'infinity'::float
+        OR
+            ${ctx.column} = '-infinity'::float
+    )
+`;
+
+const nansQueryTpl = ctx => `
+    __cdb_nans AS (
+        SELECT
+            count(*) AS __cdb_nans_count
+        FROM (${ctx.query}) __cdb_nans_query
+        WHERE ${ctx.column} = 'NaN'::float
+    )
+`;
+
+const specialNumericValuesColumnDefinitionTpl = () => `
+    __cdb_infinities_count AS infinities_count,
+    __cdb_nans_count AS nans_count
+`;
+
+const specialNumericValuesCTETpl = () => `
+    __cdb_infinities, __cdb_nans
+`;
+
+const specialNumericValuesColumnTpl = () => `
+    infinities_count, nans_count
+`;
+
+const histogramQueryTpl = ctx => `
+    SELECT
+        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
+        __cdb_bins_number AS bins_number,
+        __cdb_nulls_count AS nulls_count,
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''}
+        __cdb_avg_val AS avg_val,
+        CASE WHEN __cdb_min_val = __cdb_max_val
+            THEN 0
+            ELSE GREATEST(
+                1,
+                LEAST(
+                    WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),
+                    __cdb_bins_number
+                )
+            ) - 1
+        END AS bin,
+        min(${ctx.column})::numeric AS min,
+        max(${ctx.column})::numeric AS max,
+        avg(${ctx.column})::numeric AS avg,
+        count(*) AS freq
+    FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins
+        ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''}
+    GROUP BY bin, bins_number, bin_width, nulls_count, avg_val
+        ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''}
+    ORDER BY bin
+`;
+
+const BIN_MIN_NUMBER = 6;
+const BIN_MAX_NUMBER = 48;
+
+/**
+Numeric histogram:
+{
+    type: 'histogram',
+    options: {
+        column: 'name', // column data type: numeric
+        bins: 10 // OPTIONAL
+    }
+}
+*/
+module.exports = class NumericHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+    }
+
+    _buildQuery (psql, override, callback) {
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column,
+            isFloatColumn: this._columnType === 'float',
+            query: this.query,
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            minBins: BIN_MIN_NUMBER,
+            maxBins: BIN_MAX_NUMBER,
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+            WITH
+            ${filteredQueryTpl(ctx)},
+            ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)},
+            ${this._hasOverridenBins(ctx.override) ?
+                overrideBinsQueryTpl(ctx) :
+                `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`
+            },
+            ${nullsQueryTpl(ctx)}
+            ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''}
+            ${histogramQueryTpl(ctx)}
+        `;
+    }
+
+    _hasOverridenBins (override) {
+        return override && override.hasOwnProperty('bins');
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: this._populateBinStart(firstRow, override),
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val,
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq }));
+    }
+
+    _populateBinStart (firstRow, override = {}) {
+        let binStart;
+
+        if (override.hasOwnProperty('start')) {
+            binStart = this._getBinStart(override);
+        } else {
+            binStart = firstRow.min;
+        }
+
+        return binStart;
+    }
+
+};

lib/cartodb/models/dataview/list.js

@@ -1,11 +1,9 @@
-var dot = require('dot');
+const BaseDataview = require('./base');
-dot.templateSettings.strip = false;
+const debug = require('debug')('windshaft:dataview:list');
 
-var BaseWidget = require('./base');
+const TYPE = 'list';
 
-var TYPE = 'list';
+const listSqlTpl = ctx => `select ${ctx.columns} from (${ctx.query}) as _cdb_list`;
-
-var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
 
 /**
 {
@@ -15,52 +13,52 @@ var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as
     }
 }
 */
+module.exports = class List extends BaseDataview {
+    constructor (query, options = {}) {
+        super();
 
-function List(query, options) {
+        this._checkOptions(options);
-    options = options || {};
 
-    if (!Array.isArray(options.columns)) {
+        this.query = query;
-        throw new Error('List expects `columns` array in widget options');
+        this.columns = options.columns;
     }
 
-    BaseWidget.apply(this);
+    _checkOptions (options) {
-
+        if (!Array.isArray(options.columns)) {
-    this.query = query;
+            throw new Error('List expects `columns` array in dataview options');
-    this.columns = options.columns;
+        }
-}
-
-List.prototype = new BaseWidget();
-List.prototype.constructor = List;
-
-module.exports = List;
-
-List.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
     }
 
-    var listSql = listSqlTpl({
+    sql (psql, override, callback) {
-        _query: this.query,
+        if (!callback) {
-        _columns: this.columns.join(', ')
+            callback = override;
-    });
+        }
 
-    return callback(null, listSql);
+        const listSql = listSqlTpl({
-};
+            query: this.query,
+            columns: this.columns.join(', ')
+        });
 
-List.prototype.format = function(result) {
+        debug(listSql);
-    return {
-        rows: result.rows
-    };
-};
 
-List.prototype.getType = function() {
+        return callback(null, listSql);
-    return TYPE;
+    }
-};
 
-List.prototype.toString = function() {
+    format (result) {
-    return JSON.stringify({
+        return {
-        _type: TYPE,
+            rows: result.rows
-        _query: this.query,
+        };
-        _columns: this.columns.join(', ')
+    }
-    });
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _columns: this.columns.join(', ')
+        });
+    }
 };

lib/cartodb/server.js

@@ -4,6 +4,8 @@ var RedisPool = require('redis-mpool');
 var cartodbRedis = require('cartodb-redis');
 var _ = require('underscore');
 
+var lzmaMiddleware = require('./middleware/lzma');
+
 var controller = require('./controllers');
 
 var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
@@ -364,6 +366,8 @@ function bootstrap(opts) {
         next();
     });
 
+    app.use(lzmaMiddleware);
+
     // temporary measure until we upgrade to newer version expressjs so we can check err.status
     app.use(function(err, req, res, next) {
         if (err) {

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "3.12.10",
+  "version": "3.13.1",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -23,9 +23,9 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.58.1",
+    "camshaft": "0.59.1",
     "cartodb-psql": "0.10.1",
-    "cartodb-query-tables": "0.2.0",
+    "cartodb-query-tables": "0.3.0",
     "cartodb-redis": "0.14.0",
     "debug": "~2.2.0",
     "dot": "~1.0.2",
@@ -38,11 +38,12 @@
     "queue-async": "~1.0.7",
     "redis-mpool": "0.4.1",
     "request": "~2.79.0",
+    "semver": "~5.3.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.19.2",
+    "turbo-carto": "0.20.0",
     "underscore": "~1.6.0",
-    "windshaft": "3.3.1",
+    "windshaft": "3.3.2",
     "yargs": "~5.0.0"
   },
   "devDependencies": {
@@ -52,7 +53,6 @@
     "moment": "~2.18.1",
     "nock": "~2.11.0",
     "redis": "~0.12.1",
-    "semver": "~1.1.4",
     "strftime": "~0.8.2"
   },
   "scripts": {
@@ -62,6 +62,6 @@
   },
   "engines": {
     "node": ">=6.9",
-    "yarn": "^0.21.3"
+    "yarn": ">=0.27.5 <1.0.0"
   }
 }

test/acceptance/analysis/analysis-layers-use-cases.js

@@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client');
 var dot = require('dot');
 var debug = require('debug')('windshaft:cartodb:test');
 
-describe('analysis-layers use cases', function() {
+describe('analysis-layers use cases', function () {
 
 
-    var multitypeStyleTemplate = dot.template([
+    var multitypeStyleTemplate = dot.template(
-        "#points['mapnik::geometry_type'=1] {",
+        `#points['mapnik::geometry_type'=1] {
-        "  marker-fill-opacity: {{=it._opacity}};",
+          marker-fill-opacity: {{=it._opacity}};
-        "  marker-line-color: #FFF;",
+          marker-line-color: #FFF;
-        "  marker-line-width: 0.5;",
+          marker-line-width: 0.5;
-        "  marker-line-opacity: {{=it._opacity}};",
+          marker-line-opacity: {{=it._opacity}};
-        "  marker-placement: point;",
+          marker-placement: point;
-        "  marker-type: ellipse;",
+          marker-type: ellipse;
-        "  marker-width: 8;",
+          marker-width: 8;
-        "  marker-fill: {{=it._color}};",
+          marker-fill: {{=it._color}};
-        "  marker-allow-overlap: true;",
+          marker-allow-overlap: true;
-        "}",
+        }
-        "#lines['mapnik::geometry_type'=2] {",
+        #lines['mapnik::geometry_type'=2] {
-        "  line-color: {{=it._color}};",
+          line-color: {{=it._color}};
-        "  line-width: 2;",
+          line-width: 2;
-        "  line-opacity: {{=it._opacity}};",
+          line-opacity: {{=it._opacity}};
-        "}",
+        }
-        "#polygons['mapnik::geometry_type'=3] {",
+        #polygons['mapnik::geometry_type'=3] {
-        "  polygon-fill: {{=it._color}};",
+          polygon-fill: {{=it._color}};
-        "  polygon-opacity: {{=it._opacity}};",
+          polygon-opacity: {{=it._opacity}};
-        "  line-color: #FFF;",
+          line-color: #FFF;
-        "  line-width: 0.5;",
+          line-width: 0.5;
-        "  line-opacity: {{=it._opacity}};",
+          line-opacity: {{=it._opacity}};
-        "}"
+        }`
-    ].join('\n'));
+    );
 
 
     function cartocss(color, opacity) {
@@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() {
             version: '1.5.0',
             layers: layers,
             dataviews: dataviews || {},
-            analysis: analysis || []
+            analyses: analysis || []
         };
     }
 
-    function analysisDef(analysis) {
-        return JSON.stringify(analysis);
-    }
-
     var DEFAULT_MULTITYPE_STYLE = cartocss();
 
     var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 };
 
+    var pointInPolygonDef = {
+        id: 'a1',
+        type: 'point-in-polygon',
+        params: {
+            points_source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_rent_listings'
+                }
+            },
+            polygons_source: {
+                type: 'buffer',
+                params: {
+                    source: {
+                        type: 'source',
+                        params: {
+                            query: 'select * from analysis_banks'
+                        }
+                    },
+                    radius: 250
+                }
+            }
+        }
+    };
+
+    var bufferDef = {
+        id: 'b1',
+        type: 'buffer',
+        params: {
+            source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_banks'
+                }
+            },
+            radius: 250
+        }
+    };
+
     var useCases = [
         {
             desc: '1 mapnik layer',
@@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() {
                     {
                         type: 'cartodb',
                         options: {
-                            sql: "select * from analysis_rent_listings",
+                            sql: 'select * from analysis_rent_listings',
                             cartocss: DEFAULT_MULTITYPE_STYLE,
                             cartocss_version: '2.3.0'
                         }
@@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_banks",
+                        sql: 'select * from analysis_banks',
                         cartocss: cartocss('#2167AB'),
                         cartocss_version: '2.3.0'
                     }
@@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
@@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
+                        source: {
-                            "type": "buffer",
+                            id: 'b1'
-                            "params": {
+                        },
-                                "source": {
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                                    "type": "source",
+                        cartocss_version: '2.3.0'
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 250
-                            }
-                        }),
-                        cartocss: cartocss('black', 0.5)
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    bufferDef
+                ]
+            )
         },
 
         {
@@ -137,531 +169,115 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
+                        source: {
-                            "type": "point-in-polygon",
+                            id: 'a1'
-                            "params": {
+                        },
-                                "pointsSource": {
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                                    "type": "source",
+                        cartocss_version: '2.3.0'
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    pointInPolygonDef
+                ]
+            )
         },
 
         {
             desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                }
-            ])
-        },
-
-        {
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "buffer",
-                            "params": {
-                                "source": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 300
-                            }
-                        }),
-                        cartocss: cartocss('magenta', 0.5)
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 300
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b1" },
-                        "cartocss": cartocss('green', 1.0),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b2" },
-                        "cartocss": cartocss('magenta', 0.5),
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            [
-                {
-                    id: "b2",
-                    options: {
-                        def: analysisDef({
-                            "type": "count-in-polygon",
-                            "id": "a0",
-                            "params": {
-                                "columnName": 'count_airbnb',
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        query: "select * from analysis_rent_listings"
-                                    },
-                                    dataviews: {
-                                        price_histogram: {
-                                            type: 'histogram',
-                                            options: {
-                                                column: 'price'
-                                            }
-                                        }
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "id": "b1",
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "id": "b0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    },
-                                    dataviews: {
-                                        bank_category: {
-                                            type: 'aggregation',
-                                            options: {
-                                                column: 'bank'
-                                            }
-                                        }
-                                    }
-                                }
-                            },
-                            dataviews: {
-                                count_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'count_airbnb'
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'I. Distribution centers',
             mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "b0" },
+                            source: {
-                            "cartocss": [
+                                id: 'a1'
-                                "#distribution_centers {",
+                            },
-                                "  marker-fill-opacity: 1.0;",
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
-                                "  marker-line-color: #FFF;",
+                            cartocss_version: '2.3.0'
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: blue;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
                         }
                     },
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a0" },
+                            sql: 'select * from analysis_rent_listings',
-                            "cartocss": [
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
-                                "#shops {",
+                            cartocss_version: '2.3.0'
-                                "  marker-fill-opacity: 1.0;",
-                                "  marker-line-color: #FFF;",
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: red;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
-                        }
-                    },
-                    {
-                        type: 'cartodb',
-                        options: {
-                            "source": { id: "a1" },
-                            "cartocss": [
-                                "#routing {",
-                                "  line-color: ramp([routing_time], colorbrewer(Reds));",
-                                "  line-width: ramp([routing_time], 2, 8);",
-                                "  line-opacity: 1.0;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
                         }
                     }
                 ],
-                // dataviews
+                {},
-                {
-                    distribution_center_name_category: {
-                        source: { id: 'b0' },
-                        type: 'aggregation',
-                        options: {
-                            column: 'name'
-                        }
-                    },
-                    time_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_time'
-                        }
-                    },
-                    distance_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_distance'
-                        }
-                    }
-                },
-                // analysis
                 [
-                    {
+                    pointInPolygonDef
-                        id: 'a1',
-                        type: 'routing-n-to-n',
-                        params: {
-                            // distanceColumn: 'routing_distance',
-                            // timeColumn: 'routing_time',
-                            originSource: {
-                                id: 'b0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from distribution_centers'
-                                }
-                            },
-                            destinationSource: {
-                                id: 'a0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from shops'
-                                }
-                            }
-                        }
-                    }
                 ]
             )
         },
 
         {
-            skip: true,
+            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            desc: 'II. Population analysis',
             mapConfig: mapConfig(
-            // layers
-            [
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a2" },
-                        "cartocss": [
-                            "#count_in_polygon {",
-                            "  polygon-opacity: 1.0",
-                            "  line-color: #FFF;",
-                            "  line-width: 0.5;",
-                            "  line-opacity: 0.7",
-                            "  polygon-fill: ramp([estimated_people], colorbrewer(Reds));",
-                            "}"
-                        ].join('\n'),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a0" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            // dataviews
-            {
-                total_population_formula: {
-                    "source": { id: "a3" },
-                    type: 'formula',
-                    options: {
-                        column: 'total_population',
-                        operation: 'sum'
-                    }
-                },
-                people_histogram: { // this injects a range filter at `a2` node output
-                    "source": { id: "a2" },
-                    type: 'histogram',
-                    options: {
-                        column: 'estimated_people'
-                    }
-                },
-                subway_line_category: { // this injects a category filter at `a0` node output
-                    "source": { id: "a0" },
-                    type: 'aggregation',
-                    options: {
-                        column: 'subway_line'
-                    }
-                }
-            },
-            // analysis
-            [
-                {
-                    id: 'a3',
-                    // this will union the polygons, produce just one polygon, and calculate the total population for it
-                    type: 'total-population',
-                    params: {
-                        columnName: 'total_population',
-                        source: {
-                            id: 'a2',
-                            type: 'estimated-population',
-                            params: {
-                                columnName: 'estimated_people',
-                                source: {
-                                    id: 'a1',
-                                    type: 'trade-area',
-                                    params: {
-                                        source: {
-                                            "id": "a0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from subway_stops"
-                                            }
-                                        },
-                                        kind: 'walk',
-                                        time: 300
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'III. Point in polygon',
-            mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a1" },
+                            sql: 'select * from analysis_rent_listings',
-                            "cartocss": [
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
-                                "#count_in_polygon {",
+                            cartocss_version: '2.3.0'
-                                "  polygon-opacity: 1.0",
+                        }
-                                "  line-color: #FFF;",
+                    },
-                                "  line-width: 0.5;",
+                    {
-                                "  line-opacity: 0.7",
+                        type: 'cartodb',
-                                "  polygon-fill: ramp([count_people], colorbrewer(Reds));",
+                        options: {
-                                "}"
+                            source: {
-                            ].join('\n'),
+                                id: 'a1'
-                            "cartocss_version": "2.3.0"
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            source: {
+                                id: 'b1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     }
                 ],
-                // dataviews
+                {},
-                {
-                    age_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'age'
-                        }
-                    },
-                    income_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'income'
-                        }
-                    },
-                    gender_category: {
-                        "source": { id: "a0" },
-                        type: 'aggregation',
-                        options: {
-                            column: 'gender'
-                        }
-                    }
-                },
-                // analysis
                 [
-                    {
+                    bufferDef,
-                        "id": "a1",
+                    pointInPolygonDef
-                        "type": "count-in-polygon",
-                        "params": {
-                            "columnName": 'count_people',
-                            "pointsSource": {
-                                "id": 'a0',
-                                "type": "source",
-                                "params": {
-                                    query: "select the_geom, age, gender, income from people"
-                                }
-                            },
-                            "polygonsSource": {
-                                "id": "b0",
-                                "type": "source",
-                                "params": {
-                                    query: "select * from postal_codes"
-                                }
-                            }
-                        }
-                    }
                 ]
             )
         }
-
     ];
 
-    useCases.forEach(function(useCase, imageIdx) {
+    useCases.forEach(function (useCase) {
         if (!!useCase.skip) {
-            debug(JSON.stringify(useCase.mapConfig, null, 4));
+            return debug(JSON.stringify(useCase.mapConfig, null, 4));
         }
-        it.skip('should implement use case: "' + useCase.desc + '"', function(done) {
+        it(`should implement use case: '${useCase.desc}'`, function (done) {
 
             var testClient = new TestClient(useCase.mapConfig, 1234);
 
             var tile = useCase.tile || TILE_ANALYSIS_TABLES;
 
-            testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) {
+            testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) {
                 assert.ok(!err, err);
 
-                image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
+                //image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
 
                 assert.equal(image.width(), 256);
 

test/acceptance/named_maps_static_view.js

@@ -192,12 +192,36 @@ describe('named maps static view', function() {
             }
             getStaticMap({ zoom: 3 }, function(err, img) {
                 assert.ok(!err);
-                img.save('/tmp/static.png');
                 assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done);
             });
         });
     });
 
+    it('should return override bbox', function (done) {
+        var view = {
+            bounds: {
+                west: 0,
+                south: 0,
+                east: 45,
+                north: 45
+            },
+            zoom: 4,
+            center: {
+                lng: 40,
+                lat: 20
+            }
+        };
+        templateMaps.addTemplate(username, createTemplate(view), function (err) {
+            if (err) {
+                return done(err);
+            }
+            getStaticMap({ bbox: '0,45,90,45' }, function(err, img) {
+                assert.ok(!err);
+                assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), IMAGE_TOLERANCE, done);
+            });
+        });
+    });
+
     it('should allow to select the layers to render', function (done) {
         var view = {
             bounds: {

test/unit/cartodb/lzmaMiddleware.test.js

@@ -0,0 +1,36 @@
+var assert = require('assert');
+var testHelper = require('../../support/test_helper');
+
+var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma');
+
+describe('lzma-middleware', function() {
+
+    it('it should extend params with decoded lzma', function(done) {
+        var qo = {
+            config: {
+                version: '1.3.0'
+            }
+        };
+        testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
+            var req = {
+                headers: {
+                    host:'localhost'
+                },
+                query: {
+                    api_key: 'test',
+                    lzma: data
+                }
+            };
+            lzmaMiddleware(req, {}, function(err) {
+                if ( err ) {
+                    return done(err);
+                }
+                var query = req.query;
+                assert.deepEqual(qo.config, query.config);
+                assert.equal('test', query.api_key);
+                done();
+            });
+        });
+    });
+
+});

test/unit/cartodb/req2params.test.js

@@ -1,6 +1,6 @@
 var assert = require('assert');
 var _ = require('underscore');
-var test_helper = require('../../support/test_helper');
+require('../../support/test_helper');
 
 var RedisPool = require('redis-mpool');
 var cartodbRedis = require('cartodb-redis');
@@ -98,34 +98,31 @@ describe('req2params', function() {
       });
     });
 
-    it('it should extend params with decoded lzma', function(done) {
+    it('it should remove invalid params', function(done) {
-        var qo = {
+        var config = {
-            config: {
+            version: '1.3.0'
-                version: '1.3.0'
+        };
+        var req = {
+            headers: {
+                host:'localhost'
+            },
+            query: {
+                non_included: 'toberemoved',
+                api_key: 'test',
+                style: 'override',
+                config: config
             }
         };
-        test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
+        baseController.req2params(prepareRequest(req), function(err, req) {
-            var req = {
+            if (err) {
-                headers: {
+                return done(err);
-                    host:'localhost'
+            }
-                },
+            var query = req.params;
-                query: {
+            assert.deepEqual(config, query.config);
-                    non_included: 'toberemoved',
+            assert.equal('test', query.api_key);
-                    api_key: 'test',
+            assert.equal(undefined, query.non_included);
-                    style: 'override',
+            assert.equal(undefined, query.style);
-                    lzma: data
+            done();
-                }
-            };
-            baseController.req2params(prepareRequest(req), function(err, req) {
-                if ( err ) {
-                    return done(err);
-                }
-                var query = req.params;
-                assert.deepEqual(qo.config, query.config);
-                assert.equal('test', query.api_key);
-                assert.equal(undefined, query.non_included);
-                done();
-            });
         });
     });
 

yarn.lock

@@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1:
     mapnik "~3.5.0"
     sphericalmercator "1.0.x"
 
-abbrev@1, abbrev@1.0.x:
+abbrev@1:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f"
+
+abbrev@1.0.x:
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135"
 
@@ -28,6 +32,15 @@ ajv@^4.9.1:
     co "^4.6.0"
     json-stable-stringify "^1.0.1"
 
+ajv@^5.1.0:
+  version "5.2.2"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.2.tgz#47c68d69e86f5d953103b0074a9430dc63da5e39"
+  dependencies:
+    co "^4.6.0"
+    fast-deep-equal "^1.0.0"
+    json-schema-traverse "^0.3.0"
+    json-stable-stringify "^1.0.1"
+
 align-text@^0.1.1, align-text@^0.1.3:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
@@ -48,13 +61,9 @@ ansi-styles@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
 
-ap@~0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110"
-
 aproba@^1.0.3:
-  version "1.1.2"
+  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1"
+  resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
 
 are-we-there-yet@~1.1.2:
   version "1.1.4"
@@ -105,7 +114,11 @@ aws-sign2@~0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f"
 
-aws4@^1.2.1:
+aws-sign2@~0.7.0:
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
+
+aws4@^1.2.1, aws4@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
 
@@ -150,6 +163,18 @@ boom@2.x.x:
   dependencies:
     hoek "2.x.x"
 
+boom@4.x.x:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31"
+  dependencies:
+    hoek "4.x.x"
+
+boom@5.x.x:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02"
+  dependencies:
+    hoek "4.x.x"
+
 brace-expansion@^1.1.7:
   version "1.1.8"
   resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292"
@@ -194,9 +219,9 @@ camelcase@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
 
-camshaft@0.58.1:
+camshaft@0.59.1:
-  version "0.58.1"
+  version "0.59.1"
-  resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.1.tgz#e4156580683f624212ea3020e59790ad006f24cc"
+  resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.1.tgz#a2e87fa4a0236655160cd1a6a9e966cbbf86dd43"
   dependencies:
     async "^1.5.2"
     bunyan "1.8.1"
@@ -253,9 +278,9 @@ cartodb-psql@0.10.1, cartodb-psql@^0.10.1:
     pg cartodb/node-postgres#6.1.6-cdb1
     underscore "~1.6.0"
 
-cartodb-query-tables@0.2.0:
+cartodb-query-tables@0.3.0:
-  version "0.2.0"
+  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44"
+  resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756"
 
 cartodb-redis@0.14.0:
   version "0.14.0"
@@ -343,12 +368,16 @@ combined-stream@^1.0.5, combined-stream@~1.0.5:
   dependencies:
     delayed-stream "~1.0.0"
 
-commander@2.9.0, commander@^2.9.0:
+commander@2.9.0:
   version "2.9.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"
   dependencies:
     graceful-readlink ">= 1.0.0"
 
+commander@^2.9.0:
+  version "2.11.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563"
+
 concat-map@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -368,8 +397,8 @@ content-disposition@0.5.1:
   resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.1.tgz#87476c6a67c8daa87e32e87616df883ba7fb071b"
 
 content-type@~1.0.1:
-  version "1.0.2"
+  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.2.tgz#b7d113aee7a8dd27bd21133c4dc2529df1721eed"
+  resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
 
 cookie-signature@1.0.6:
   version "1.0.6"
@@ -389,6 +418,12 @@ cryptiles@2.x.x:
   dependencies:
     boom "2.x.x"
 
+cryptiles@3.x.x:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe"
+  dependencies:
+    boom "5.x.x"
+
 d3-queue@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2"
@@ -403,7 +438,7 @@ date-now@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
 
-debug@2.2.0, debug@^2.2.0, debug@~2.2.0:
+debug@2.2.0, debug@~2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da"
   dependencies:
@@ -421,6 +456,12 @@ debug@^1.0.4:
   dependencies:
     ms "2.0.0"
 
+debug@^2.2.0:
+  version "2.6.9"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
+  dependencies:
+    ms "2.0.0"
+
 decamelize@^1.0.0, decamelize@^1.1.1:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -491,7 +532,11 @@ domutils@1.5:
     dom-serializer "0"
     domelementtype "1"
 
-dot@^1.0.3, dot@~1.0.2:
+dot@^1.0.3:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9"
+
+dot@~1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427"
 
@@ -602,7 +647,7 @@ express@~4.13.3:
     utils-merge "1.0.0"
     vary "~1.0.1"
 
-extend@~3.0.0:
+extend@~3.0.0, extend@~3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444"
 
@@ -610,6 +655,10 @@ extsprintf@1.3.0, extsprintf@^1.2.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05"
 
+fast-deep-equal@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff"
+
 fast-levenshtein@~2.0.4:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
@@ -648,9 +697,17 @@ form-data@~2.1.1:
     combined-stream "^1.0.5"
     mime-types "^2.1.12"
 
+form-data@~2.3.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf"
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.5"
+    mime-types "^2.1.12"
+
 forwarded@~0.1.0:
-  version "0.1.0"
+  version "0.1.2"
-  resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.0.tgz#19ef9874c4ae1c297bcf078fde63a09b66a84363"
+  resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"
 
 fresh@0.3.0:
   version "0.3.0"
@@ -691,11 +748,11 @@ gauge@~2.7.3:
     wide-align "^1.1.0"
 
 gdal@~0.9.2:
-  version "0.9.4"
+  version "0.9.6"
-  resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.4.tgz#6dad4abb8ffb3e0d51150fb7935ad7c622c81818"
+  resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.6.tgz#0cf75d830d35847b4274368b10e04a925321a0ba"
   dependencies:
-    nan "~2.5.0"
+    nan "~2.6.2"
-    node-pre-gyp "~0.6.27"
+    node-pre-gyp "~0.6.36"
 
 generate-function@^2.0.0:
   version "2.0.0"
@@ -733,7 +790,7 @@ getpass@^0.1.1:
   dependencies:
     assert-plus "^1.0.0"
 
-glob@7.1.1, glob@^7.0.5, glob@^7.1.1:
+glob@7.1.1:
   version "7.1.1"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8"
   dependencies:
@@ -764,6 +821,17 @@ glob@^6.0.1:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
+glob@^7.0.5, glob@^7.1.1:
+  version "7.1.2"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
+  dependencies:
+    fs.realpath "^1.0.0"
+    inflight "^1.0.4"
+    inherits "2"
+    minimatch "^3.0.4"
+    once "^1.3.0"
+    path-is-absolute "^1.0.0"
+
 graceful-fs@^4.1.2:
   version "4.1.11"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
@@ -804,6 +872,10 @@ har-schema@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e"
 
+har-schema@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
+
 har-validator@~2.0.6:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d"
@@ -820,6 +892,13 @@ har-validator@~4.2.1:
     ajv "^4.9.1"
     har-schema "^1.0.5"
 
+har-validator@~5.0.3:
+  version "5.0.3"
+  resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd"
+  dependencies:
+    ajv "^5.1.0"
+    har-schema "^2.0.0"
+
 has-ansi@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91"
@@ -834,7 +913,7 @@ has-unicode@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9"
 
-hawk@~3.1.3:
+hawk@3.1.3, hawk@~3.1.3:
   version "3.1.3"
   resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4"
   dependencies:
@@ -843,6 +922,15 @@ hawk@~3.1.3:
     hoek "2.x.x"
     sntp "1.x.x"
 
+hawk@~6.0.2:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038"
+  dependencies:
+    boom "4.x.x"
+    cryptiles "3.x.x"
+    hoek "4.x.x"
+    sntp "2.x.x"
+
 hiredis@~0.5.0:
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87"
@@ -854,6 +942,10 @@ hoek@2.x.x:
   version "2.16.3"
   resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed"
 
+hoek@4.x.x:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d"
+
 hosted-git-info@^2.1.4:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c"
@@ -883,6 +975,14 @@ http-signature@~1.1.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
+http-signature@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
+  dependencies:
+    assert-plus "^1.0.0"
+    jsprim "^1.2.2"
+    sshpk "^1.7.0"
+
 husl@^6.0.1:
   version "6.0.6"
   resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d"
@@ -935,8 +1035,8 @@ is-fullwidth-code-point@^1.0.0:
     number-is-nan "^1.0.0"
 
 is-my-json-valid@^2.12.4:
-  version "2.16.0"
+  version "2.16.1"
-  resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.0.tgz#f079dd9bfdae65ee2038aae8acbc86ab109e3693"
+  resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11"
   dependencies:
     generate-function "^2.0.0"
     generate-object-property "^1.1.0"
@@ -991,16 +1091,16 @@ istanbul@~0.4.3:
     wordwrap "^1.0.0"
 
 js-base64@^2.1.9:
-  version "2.1.9"
+  version "2.3.2"
-  resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce"
+  resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.3.2.tgz#a79a923666372b580f8e27f51845c6f7e8fbfbaf"
 
 js-string-escape@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef"
 
 js-yaml@3.x, js-yaml@^3.4.6:
-  version "3.9.1"
+  version "3.10.0"
-  resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.9.1.tgz#08775cebdfdd359209f0d2acd383c8f86a6904a0"
+  resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc"
   dependencies:
     argparse "^1.0.7"
     esprima "^4.0.0"
@@ -1022,6 +1122,10 @@ jshint@~2.9.4:
     shelljs "0.3.x"
     strip-json-comments "1.0.x"
 
+json-schema-traverse@^0.3.0:
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340"
+
 json-schema@0.2.3:
   version "0.2.3"
   resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
@@ -1231,17 +1335,17 @@ millstone@0.6.17:
     underscore "~1.6.0"
     zipfile "~0.5.5"
 
-mime-db@~1.29.0:
+mime-db@~1.30.0:
-  version "1.29.0"
+  version "1.30.0"
-  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.29.0.tgz#48d26d235589651704ac5916ca06001914266878"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01"
 
-mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7:
+mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, mime-types@~2.1.7:
-  version "2.1.16"
+  version "2.1.17"
-  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.16.tgz#2b858a52e5ecd516db897ac2be87487830698e23"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a"
   dependencies:
-    mime-db "~1.29.0"
+    mime-db "~1.30.0"
 
-mime@1.3.4, mime@~1.3.4:
+mime@1.3.4:
   version "1.3.4"
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53"
 
@@ -1249,13 +1353,17 @@ mime@~1.2.11:
   version "1.2.11"
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10"
 
-"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2:
+mime@~1.3.4:
+  version "1.3.6"
+  resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0"
+
+"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
   dependencies:
     brace-expansion "^1.1.7"
 
-minimist@0.0.8, minimist@~0.0.1:
+minimist@0.0.8:
   version "0.0.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
 
@@ -1263,6 +1371,10 @@ minimist@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
 
+minimist@~0.0.1:
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf"
+
 minimist@~0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce"
@@ -1313,17 +1425,17 @@ mv@~2:
     ncp "~2.0.0"
     rimraf "~2.4.0"
 
-nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.6.2:
+nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.7.0:
-  version "2.6.2"
+  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46"
 
 nan@~2.4.0:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232"
 
-nan@~2.5.0:
+nan@~2.6.2:
-  version "2.5.1"
+  version "2.6.2"
-  resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45"
 
 ncp@~2.0.0:
   version "2.0.0"
@@ -1344,15 +1456,16 @@ nock@~2.11.0:
     mkdirp "^0.5.0"
     propagate "0.3.x"
 
-node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36:
+node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38:
-  version "0.6.36"
+  version "0.6.38"
-  resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786"
+  resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.38.tgz#e92a20f83416415bb4086f6d1fb78b3da73d113d"
   dependencies:
+    hawk "3.1.3"
     mkdirp "^0.5.1"
     nopt "^4.0.1"
     npmlog "^4.0.2"
     rc "^1.1.7"
-    request "^2.81.0"
+    request "2.81.0"
     rimraf "^2.6.1"
     semver "^5.3.0"
     tar "^2.2.1"
@@ -1397,7 +1510,7 @@ number-is-nan@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
 
-oauth-sign@~0.8.1:
+oauth-sign@~0.8.1, oauth-sign@~0.8.2:
   version "0.8.2"
   resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
 
@@ -1475,8 +1588,8 @@ parse-json@^2.2.0:
     error-ex "^1.2.0"
 
 parseurl@~1.3.1:
-  version "1.3.1"
+  version "1.3.2"
-  resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56"
+  resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3"
 
 path-exists@^2.0.0:
   version "2.1.0"
@@ -1504,6 +1617,10 @@ performance-now@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5"
 
+performance-now@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
+
 pg-connection-string@0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7"
@@ -1516,10 +1633,9 @@ pg-pool@1.*:
     object-assign "4.1.0"
 
 pg-types@1.*:
-  version "1.12.0"
+  version "1.12.1"
-  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0"
+  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2"
   dependencies:
-    ap "~0.2.0"
     postgres-array "~1.0.0"
     postgres-bytea "~1.0.0"
     postgres-date "~1.0.0"
@@ -1660,6 +1776,10 @@ qs@~6.4.0:
   version "6.4.0"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233"
 
+qs@~6.5.1:
+  version "6.5.1"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8"
+
 queue-async@~1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945"
@@ -1700,7 +1820,7 @@ read-pkg@^1.0.0:
     normalize-package-data "^2.3.2"
     path-type "^1.0.0"
 
-readable-stream@1.1, readable-stream@~1.1.9:
+readable-stream@1.1:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e"
   dependencies:
@@ -1730,6 +1850,15 @@ readable-stream@~1.0.2:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
+readable-stream@~1.1.9:
+  version "1.1.14"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.1"
+    isarray "0.0.1"
+    string_decoder "~0.10.x"
+
 redis-mpool@0.4.1, redis-mpool@~0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80"
@@ -1747,32 +1876,7 @@ repeat-string@^1.5.2:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
 
-request@2.x, request@^2.55.0, request@~2.79.0:
+request@2.81.0:
-  version "2.79.0"
-  resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
-  dependencies:
-    aws-sign2 "~0.6.0"
-    aws4 "^1.2.1"
-    caseless "~0.11.0"
-    combined-stream "~1.0.5"
-    extend "~3.0.0"
-    forever-agent "~0.6.1"
-    form-data "~2.1.1"
-    har-validator "~2.0.6"
-    hawk "~3.1.3"
-    http-signature "~1.1.0"
-    is-typedarray "~1.0.0"
-    isstream "~0.1.2"
-    json-stringify-safe "~5.0.1"
-    mime-types "~2.1.7"
-    oauth-sign "~0.8.1"
-    qs "~6.3.0"
-    stringstream "~0.0.4"
-    tough-cookie "~2.3.0"
-    tunnel-agent "~0.4.1"
-    uuid "^3.0.0"
-
-request@^2.69.0, request@^2.81.0:
   version "2.81.0"
   resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0"
   dependencies:
@@ -1799,6 +1903,58 @@ request@^2.69.0, request@^2.81.0:
     tunnel-agent "^0.6.0"
     uuid "^3.0.0"
 
+request@2.x, request@^2.55.0, request@^2.69.0:
+  version "2.82.0"
+  resolved "https://registry.yarnpkg.com/request/-/request-2.82.0.tgz#2ba8a92cd7ac45660ea2b10a53ae67cd247516ea"
+  dependencies:
+    aws-sign2 "~0.7.0"
+    aws4 "^1.6.0"
+    caseless "~0.12.0"
+    combined-stream "~1.0.5"
+    extend "~3.0.1"
+    forever-agent "~0.6.1"
+    form-data "~2.3.1"
+    har-validator "~5.0.3"
+    hawk "~6.0.2"
+    http-signature "~1.2.0"
+    is-typedarray "~1.0.0"
+    isstream "~0.1.2"
+    json-stringify-safe "~5.0.1"
+    mime-types "~2.1.17"
+    oauth-sign "~0.8.2"
+    performance-now "^2.1.0"
+    qs "~6.5.1"
+    safe-buffer "^5.1.1"
+    stringstream "~0.0.5"
+    tough-cookie "~2.3.2"
+    tunnel-agent "^0.6.0"
+    uuid "^3.1.0"
+
+request@~2.79.0:
+  version "2.79.0"
+  resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
+  dependencies:
+    aws-sign2 "~0.6.0"
+    aws4 "^1.2.1"
+    caseless "~0.11.0"
+    combined-stream "~1.0.5"
+    extend "~3.0.0"
+    forever-agent "~0.6.1"
+    form-data "~2.1.1"
+    har-validator "~2.0.6"
+    hawk "~3.1.3"
+    http-signature "~1.1.0"
+    is-typedarray "~1.0.0"
+    isstream "~0.1.2"
+    json-stringify-safe "~5.0.1"
+    mime-types "~2.1.7"
+    oauth-sign "~0.8.1"
+    qs "~6.3.0"
+    stringstream "~0.0.4"
+    tough-cookie "~2.3.0"
+    tunnel-agent "~0.4.1"
+    uuid "^3.0.0"
+
 require-directory@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
@@ -1818,8 +1974,8 @@ right-align@^0.1.1:
     align-text "^0.1.1"
 
 rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1:
-  version "2.6.1"
+  version "2.6.2"
-  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d"
+  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36"
   dependencies:
     glob "^7.0.5"
 
@@ -1829,7 +1985,7 @@ rimraf@~2.4.0:
   dependencies:
     glob "^6.0.1"
 
-safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
 
@@ -1845,10 +2001,6 @@ semver@4.3.2:
   version "4.3.2"
   resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7"
 
-semver@~1.1.4:
-  version "1.1.4"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540"
-
 semver@~4.3.3:
   version "4.3.6"
   resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da"
@@ -1857,6 +2009,10 @@ semver@~5.0.3:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a"
 
+semver@~5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
+
 send@0.13.1:
   version "0.13.1"
   resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7"
@@ -1921,6 +2077,12 @@ sntp@1.x.x:
   dependencies:
     hoek "2.x.x"
 
+sntp@2.x.x:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b"
+  dependencies:
+    hoek "4.x.x"
+
 source-map@^0.4.4:
   version "0.4.4"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b"
@@ -1928,8 +2090,8 @@ source-map@^0.4.4:
     amdefine ">=0.0.4"
 
 source-map@^0.5.1, source-map@^0.5.6, source-map@~0.5.1:
-  version "0.5.6"
+  version "0.5.7"
-  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412"
+  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
 
 source-map@~0.2.0:
   version "0.2.0"
@@ -1955,10 +2117,14 @@ speedometer@~0.1.2:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d"
 
-sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4:
+sphericalmercator@1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e"
 
+sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980"
+
 split@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9"
@@ -1970,11 +2136,11 @@ sprintf-js@~1.0.2:
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
 
 "sqlite3@2.x || 3.x":
-  version "3.1.9"
+  version "3.1.12"
-  resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.9.tgz#b2e7fbaa348380318d3834323918c3c351b8bf18"
+  resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.12.tgz#2b3a14b17162e39e8aa6e1e2487a41d0795396d8"
   dependencies:
-    nan "~2.6.2"
+    nan "~2.7.0"
-    node-pre-gyp "~0.6.36"
+    node-pre-gyp "~0.6.38"
 
 srs@1.x:
   version "1.2.0"
@@ -2036,7 +2202,7 @@ string_decoder@~1.0.3:
   dependencies:
     safe-buffer "~5.1.0"
 
-stringstream@~0.0.4:
+stringstream@~0.0.4, stringstream@~0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
 
@@ -2141,9 +2307,9 @@ torque.js@~2.11.0:
   dependencies:
     carto CartoDB/carto#0.15.1-cdb1
 
-tough-cookie@~2.3.0:
+tough-cookie@~2.3.0, tough-cookie@~2.3.2:
-  version "2.3.2"
+  version "2.3.3"
-  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.2.tgz#f081f76e4c85720e6c37a5faced737150d84072a"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561"
   dependencies:
     punycode "^1.4.1"
 
@@ -2157,9 +2323,9 @@ tunnel-agent@~0.4.1:
   version "0.4.3"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb"
 
-turbo-carto@0.19.2:
+turbo-carto@0.20.0:
-  version "0.19.2"
+  version "0.20.0"
-  resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd"
+  resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.0.tgz#ebeb4620f3e26ed3cce53a9ff30f087f46a2e3fd"
   dependencies:
     cartocolor "4.0.0"
     colorbrewer "1.0.0"
@@ -2234,7 +2400,7 @@ utils-merge@1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8"
 
-uuid@^3.0.0:
+uuid@^3.0.0, uuid@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"
 
@@ -2281,9 +2447,9 @@ window-size@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
 
-windshaft@3.3.1:
+windshaft@3.3.2:
-  version "3.3.1"
+  version "3.3.2"
-  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.1.tgz#b5557fa6b0cfa13920904f57206b86f7aa054f74"
+  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.2.tgz#72efe0dbc0d8d4bcba4211fdabd15dd2e0799df9"
   dependencies:
     abaculus cartodb/abaculus#2.0.3-cdb1
     canvas cartodb/node-canvas#1.6.2-cdb2