diff --git a/.gitignore b/.gitignore index aae26829..b39f2801 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ config.status* config/environments/*.js .idea .vscode +.nvmrc tools/munin/windshaft.conf logs/ pids/ diff --git a/HOWTO_RELEASE b/HOWTO_RELEASE index eedee1d0..b4a1c53d 100644 --- a/HOWTO_RELEASE +++ b/HOWTO_RELEASE @@ -1,11 +1,10 @@ 1. Test (make clean all check), fix if broken before proceeding 2. Ensure proper version in package.json 3. Ensure NEWS section exists for the new version, review it, add release date -4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap` +4. Recreate npm-shrinkwrap.json with: `make shrinkwrap` 5. Commit package.json, npm-shrinwrap.json, NEWS 6. git tag -a Major.Minor.Patch # use NEWS section as content -7. Announce on cartodb@googlegroups.com -8. Stub NEWS/package for next version +7. Stub NEWS/package for next version Versions: diff --git a/Makefile b/Makefile index 236a6749..5929e084 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,13 @@ all: @$(SHELL) ./scripts/install.sh clean: - rm -rf node_modules/* + rm -rf node_modules/ + +shrinkwrap: clean + rm npm-shrinkwrap.json + npm install --no-shrinkwrap --production + npm prune + npm shrinkwrap distclean: clean rm config.status* diff --git a/NEWS.md b/NEWS.md index aa57784b..2b9037fe 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,38 @@ # Changelog -## 2.87.6 +## 2.88.4 Released 2017-mm-dd +## 2.88.3 +Released 2017-03-02 + +Bug fixes: +- Category dataviews now uses the proper aggregation function for the 'Other' category. See https://github.com/CartoDB/Windshaft-cartodb/issues/628 + +## 2.88.2 +Released 2017-02-23 + +Announcements: + - Upgrades camshaft to [0.50.2](https://github.com/CartoDB/camshaft/releases/tag/0.50.2). + + +## 2.88.1 +Released 2017-02-21 + +Announcements: + - Upgrades camshaft to [0.50.1](https://github.com/CartoDB/camshaft/releases/tag/0.50.1) + + +## 2.88.0 +Released 2017-02-21 + +Announcements: + - Upgrades camshaft to [0.50.0](https://github.com/CartoDB/camshaft/releases/tag/0.50.0). + - Upgrades cartodb-psql to [0.7.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.7.1). + - Upgrades windshaft to [2.7.0](https://github.com/CartoDB/windshaft/releases/tag/2.7.0). + + ## 2.87.5 Released 2017-02-02 @@ -30,6 +59,7 @@ Released 2016-12-19 - Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242 + ## 2.87.1 Released 2016-12-13 diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b4cc67cc..f7215943 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -338,6 +338,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) { var self = this; + req.profiler.done('res'); + res.set('Cache-Control', 'public,max-age=31536000'); // Set Last-Modified header diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index a80efbc7..c15f0506 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -48,7 +48,8 @@ var rankedAggregationQueryTpl = dot.template([ ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', + ' count, categories_count', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', 'GROUP BY nulls_count, min_val, max_val, count, categories_count' @@ -129,27 +130,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { if (!!override.ownFilter) { aggregationSql = [ - "WITH", - [ - summaryQueryTpl({ - _query: _query, - _column: this.column - }), - rankedCategoriesQueryTpl({ - _query: _query, - _column: this.column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: _query, - _column: this.column - }), - categoriesSummaryCountQueryTpl({ - _query: _query, - _column: this.column - }) - ].join(',\n'), + this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), aggregationQueryTpl({ _query: _query, _column: this.column, @@ -159,30 +140,11 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } else { aggregationSql = [ - "WITH", - [ - summaryQueryTpl({ - _query: _query, - _column: this.column - }), - rankedCategoriesQueryTpl({ - _query: _query, - _column: this.column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: _query, - _column: this.column - }), - categoriesSummaryCountQueryTpl({ - _query: _query, - _column: this.column - }) - ].join(',\n'), + this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), rankedAggregationQueryTpl({ _query: _query, _column: this.column, + _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', _limit: CATEGORIES_LIMIT }) ].join('\n'); @@ -193,6 +155,32 @@ Aggregation.prototype.sql = function(psql, override, callback) { return callback(null, aggregationSql); }; +Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { + return [ + "WITH", + [ + summaryQueryTpl({ + _query: query, + _column: column + }), + rankedCategoriesQueryTpl({ + _query: query, + _column: column, + _aggregation: this.getAggregationSql(), + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + categoriesSummaryMinMaxQueryTpl({ + _query: query, + _column: column + }), + categoriesSummaryCountQueryTpl({ + _query: query, + _column: column + }) + ].join(',\n') + ].join('\n'); +}; + var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); Aggregation.prototype.getAggregationSql = function() { return aggregationFnQueryTpl({ diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index b9487095..46efd100 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -4,6 +4,13 @@ var dot = require('dot'); dot.templateSettings.strip = false; var queue = require('queue-async'); var PSQL = require('cartodb-psql'); +/** + * cartodb-psql creates `global.Promise` as an empty constructor. + * However, `turbo-carto` relies on a polyfil that fails to create the polyfil + * as it finds `global.Promise` but it doesn't find `Promise.resolve`. + */ +global.Promise = global.Promise || function() {}; +global.Promise.resolve = global.Promise.resolve || function() {}; var turboCarto = require('turbo-carto'); var SubstitutionTokens = require('../../../utils/substitution-tokens'); diff --git a/package.json b/package.json index 8864bd9f..63d5349e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.87.6", + "version": "2.88.4", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.49.0", + "camshaft": "0.50.2", "cartodb-psql": "0.7.1", "cartodb-query-tables": "0.2.0", "cartodb-redis": "0.13.2", diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 9fc9a219..259cd2af 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -1,5 +1,4 @@ require('../../support/test_helper'); - var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); @@ -107,4 +106,43 @@ describe('aggregations happy cases', function() { }); }); }); + + var operations_and_values = {'count': 9, 'sum': 45, 'avg': 5, 'max': 9, 'min': 1}; + + var query_other = [ + 'select generate_series(1,3) as val, \'other_a\' as cat, NULL as the_geom_webmercator', + 'select generate_series(4,6) as val, \'other_b\' as cat, NULL as the_geom_webmercator', + 'select generate_series(7,9) as val, \'other_c\' as cat, NULL as the_geom_webmercator', + 'select generate_series(10,12) as val, \'category_1\' as cat, NULL as the_geom_webmercator', + 'select generate_series(10,12) as val, \'category_2\' as cat, NULL as the_geom_webmercator', + 'select generate_series(10,12) as val, \'category_3\' as cat, NULL as the_geom_webmercator', + 'select generate_series(10,12) as val, \'category_4\' as cat, NULL as the_geom_webmercator', + 'select generate_series(10,12) as val, \'category_5\' as cat, NULL as the_geom_webmercator' + ].join(' UNION ALL '); + + Object.keys(operations_and_values).forEach(function (operation) { + var description = 'should aggregate OTHER category using "' + operation + '"'; + + it(description, function (done) { + this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val')); + this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) { + assert.ifError(err); + + assert.ok(aggregation); + assert.equal(aggregation.type, 'aggregation'); + assert.ok(aggregation.categories); + assert.equal(aggregation.categoriesCount, 8); + assert.equal(aggregation.count, 24); + assert.equal(aggregation.nulls, 0); + + var aggregated_categories = aggregation.categories.filter( function(category) { + return category.agg === true; + }); + assert.equal(aggregated_categories.length, 1); + assert.equal(aggregated_categories[0].value, operations_and_values[operation]); + + done(); + }); + }); + }); });