Merge branch 'node-v6' of github.com:CartoDB/Windshaft-cartodb into node-v6

.gitignore

@@ -3,6 +3,7 @@ config.status*
 config/environments/*.js
 .idea
 .vscode
+.nvmrc
 tools/munin/windshaft.conf
 logs/
 pids/

HOWTO_RELEASE

@@ -1,11 +1,10 @@
 1. Test (make clean all check), fix if broken before proceeding
 2. Ensure proper version in package.json
 3. Ensure NEWS section exists for the new version, review it, add release date
-4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap`
+4. Recreate npm-shrinkwrap.json with: `make shrinkwrap`
 5. Commit package.json, npm-shrinwrap.json, NEWS
 6. git tag -a Major.Minor.Patch # use NEWS section as content
-7. Announce on cartodb@googlegroups.com
+7. Stub NEWS/package for next version
-8. Stub NEWS/package for next version
 
 Versions:
 

Makefile

@@ -7,7 +7,13 @@ all:
 	@$(SHELL) ./scripts/install.sh
 
 clean:
-	rm -rf node_modules/*
+	rm -rf node_modules/
+
+shrinkwrap: clean
+	rm npm-shrinkwrap.json
+	npm install --no-shrinkwrap --production
+	npm prune
+	npm shrinkwrap
 
 distclean: clean
 	rm config.status*

NEWS.md

@@ -1,9 +1,38 @@
 # Changelog
 
-## 2.87.6
+## 2.88.4
 Released 2017-mm-dd
 
 
+## 2.88.3
+Released 2017-03-02
+
+Bug fixes:
+- Category dataviews now uses the proper aggregation function for the 'Other' category. See https://github.com/CartoDB/Windshaft-cartodb/issues/628
+
+## 2.88.2
+Released 2017-02-23
+
+Announcements:
+ - Upgrades camshaft to [0.50.2](https://github.com/CartoDB/camshaft/releases/tag/0.50.2).
+
+
+## 2.88.1
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.1](https://github.com/CartoDB/camshaft/releases/tag/0.50.1)
+
+
+## 2.88.0
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.0](https://github.com/CartoDB/camshaft/releases/tag/0.50.0).
+ - Upgrades cartodb-psql to [0.7.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.7.1).
+ - Upgrades windshaft to [2.7.0](https://github.com/CartoDB/windshaft/releases/tag/2.7.0).
+
+
 ## 2.87.5
 Released 2017-02-02
 
@@ -30,6 +59,7 @@ Released 2016-12-19
 
 - Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242
 
+
 ## 2.87.1
 Released 2016-12-13
 

lib/cartodb/controllers/layergroup.js

@@ -338,6 +338,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
 LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
     var self = this;
 
+    req.profiler.done('res');
+
     res.set('Cache-Control', 'public,max-age=31536000');
 
     // Set Last-Modified header

lib/cartodb/models/dataview/aggregation.js

@@ -48,7 +48,8 @@ var rankedAggregationQueryTpl = dot.template([
     '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
-    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
+    'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
+    '       count, categories_count',
     '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
@@ -129,27 +130,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
 
     if (!!override.ownFilter) {
         aggregationSql = [
-            "WITH",
+            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql(),
-                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
-                }),
-                categoriesSummaryMinMaxQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                categoriesSummaryCountQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
             aggregationQueryTpl({
                 _query: _query,
                 _column: this.column,
@@ -159,30 +140,11 @@ Aggregation.prototype.sql = function(psql, override, callback) {
         ].join('\n');
     } else {
         aggregationSql = [
-            "WITH",
+            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql(),
-                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
-                }),
-                categoriesSummaryMinMaxQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                categoriesSummaryCountQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
             rankedAggregationQueryTpl({
                 _query: _query,
                 _column: this.column,
+                _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
                 _limit: CATEGORIES_LIMIT
             })
         ].join('\n');
@@ -193,6 +155,32 @@ Aggregation.prototype.sql = function(psql, override, callback) {
     return callback(null, aggregationSql);
 };
 
+Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
+    return [
+                "WITH",
+                [
+                    summaryQueryTpl({
+                        _query: query,
+                        _column: column
+                    }),
+                    rankedCategoriesQueryTpl({
+                        _query: query,
+                        _column: column,
+                        _aggregation: this.getAggregationSql(),
+                        _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
+                    }),
+                    categoriesSummaryMinMaxQueryTpl({
+                        _query: query,
+                        _column: column
+                    }),
+                    categoriesSummaryCountQueryTpl({
+                        _query: query,
+                        _column: column
+                    })
+                ].join(',\n')
+            ].join('\n');
+};
+
 var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
 Aggregation.prototype.getAggregationSql = function() {
     return aggregationFnQueryTpl({

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -4,6 +4,13 @@ var dot = require('dot');
 dot.templateSettings.strip = false;
 var queue = require('queue-async');
 var PSQL = require('cartodb-psql');
+/**
+ * cartodb-psql creates `global.Promise` as an empty constructor.
+ * However, `turbo-carto` relies on a polyfil that fails to create the polyfil
+ * as it finds `global.Promise` but it doesn't find `Promise.resolve`.
+ */
+global.Promise = global.Promise || function() {};
+global.Promise.resolve = global.Promise.resolve || function() {};
 var turboCarto = require('turbo-carto');
 
 var SubstitutionTokens = require('../../../utils/substitution-tokens');

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.87.6",
+  "version": "2.88.4",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,7 +20,7 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.49.0",
+    "camshaft": "0.50.2",
     "cartodb-psql": "0.7.1",
     "cartodb-query-tables": "0.2.0",
     "cartodb-redis": "0.13.2",

test/acceptance/dataviews/aggregation.js

@@ -1,5 +1,4 @@
 require('../../support/test_helper');
-
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
@@ -107,4 +106,43 @@ describe('aggregations happy cases', function() {
             });
         });
     });
+
+    var operations_and_values = {'count': 9, 'sum': 45, 'avg': 5, 'max': 9, 'min': 1};
+
+    var query_other = [
+        'select generate_series(1,3) as val, \'other_a\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(4,6) as val, \'other_b\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(7,9) as val, \'other_c\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_1\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_2\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_3\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_4\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_5\' as cat, NULL as the_geom_webmercator'
+    ].join(' UNION ALL ');
+
+    Object.keys(operations_and_values).forEach(function (operation) {
+        var description = 'should aggregate OTHER category using "' + operation + '"';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 8);
+                assert.equal(aggregation.count, 24);
+                assert.equal(aggregation.nulls, 0);
+
+                var aggregated_categories = aggregation.categories.filter( function(category) {
+                    return category.agg === true;
+                });
+                assert.equal(aggregated_categories.length, 1);
+                assert.equal(aggregated_categories[0].value, operations_and_values[operation]);
+
+                done();
+            });
+        });
+    });
 });