diff --git a/NEWS.md b/NEWS.md index f3cde6ab..830b7de4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,18 +1,24 @@ # Changelog -## 8.0.0 +## 8.0.1 Released 2019-mm-dd +## 8.0.0 +Released 2019-11-13 + Breaking changes: - Schema change for "routes" in configuration file, each "router" is now an array instead of an object. See [`dd06de2`](https://github.com/CartoDB/Windshaft-cartodb/pull/1126/commits/dd06de2632661e19d64c9fbc2be0ba1a8059f54c) for more details. Announcements: - +- Added validation to only allow "count" and "sum" aggregations in dataview overview. - Added mechanism to inject custom middlewares through configuration. - Stop requiring unused config properties: "base_url", "base_url_mapconfig", and "base_url_templated". - Upgraded cartodb-query-tables to version [0.7.0](https://github.com/CartoDB/node-cartodb-query-tables/blob/0.7.0/NEWS.md#version-0.7.0). - Be able to set a coherent TTL in Cache-Control header to expire all resources belonging to a map simultaneously. - When `cache buster` in request path is `0` set header `Last-Modified` to now, it avoids stalled content in 3rd party cache providers when they add `If-Modified-Since` header into the request. +- Adding a logger to MapStore (#1134) +- Qualify calls to cartodb extension so having it in the search_path isn't necessary. +- Fix multiple DB login issues. ## 7.2.0 Released 2019-09-30 diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 7004c9ec..c7c3b97e 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: undefined + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>' diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 3e75a897..5b6a632b 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: 'logs/node-windshaft.log' + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'cartodb_user_<%= user_id %>' diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 49e08ac7..1aa59d02 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: 'logs/node-windshaft.log' + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>' diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 04797ae9..d8207730 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: '/tmp/node-windshaft.log' + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>' diff --git a/lib/api/api-router.js b/lib/api/api-router.js index f0ba1056..ff37edd7 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -85,9 +85,13 @@ module.exports = class ApiRouter { const metadataBackend = cartodbRedis({ pool: redisPool }); const pgConnection = new PgConnection(metadataBackend); + const windshaftLogger = environmentOptions.log_windshaft && global.log4js ? + global.log4js.getLogger('[windshaft]') : + null; const mapStore = new windshaft.storage.MapStore({ pool: redisPool, - expire_time: serverOptions.grainstore.default_layergroup_ttl + expire_time: serverOptions.grainstore.default_layergroup_ttl, + logger: windshaftLogger }); const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); diff --git a/lib/backends/dataview.js b/lib/backends/dataview.js index bd5a1428..994f89f3 100644 --- a/lib/backends/dataview.js +++ b/lib/backends/dataview.js @@ -8,7 +8,7 @@ var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory const dbParamsFromReqParams = require('../utils/database-params'); var OverviewsQueryRewriter = require('../utils/overviews-query-rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ - zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' + zoom_level: 'cartodb.CDB_ZoomFromScale(!scale_denominator!)' }); var dot = require('dot'); diff --git a/lib/backends/overviews-metadata.js b/lib/backends/overviews-metadata.js index 55bbb364..f82e2887 100644 --- a/lib/backends/overviews-metadata.js +++ b/lib/backends/overviews-metadata.js @@ -12,9 +12,9 @@ OverviewsMetadataBackend.prototype.getOverviewsMetadata = function (username, sq // FIXME: Currently using internal function _cdb_schema_name // CDB_Overviews should provide the schema information directly. const query = ` - SELECT *, _cdb_schema_name(base_table) - FROM CDB_Overviews( - CDB_QueryTablesText($windshaft$${queryUtils.substituteDummyTokens(sql)}$windshaft$) + SELECT *, cartodb._cdb_schema_name(base_table) + FROM cartodb.CDB_Overviews( + cartodb.CDB_QueryTablesText($windshaft$${queryUtils.substituteDummyTokens(sql)}$windshaft$) ); `; this.pgQueryRunner.run(username, query, function handleOverviewsRows (err, rows) { diff --git a/lib/backends/pg-connection.js b/lib/backends/pg-connection.js index 23464c67..5d8c9d29 100644 --- a/lib/backends/pg-connection.js +++ b/lib/backends/pg-connection.js @@ -3,6 +3,7 @@ var PSQL = require('cartodb-psql'); var _ = require('underscore'); const debug = require('debug')('cachechan'); +const dbParamsFromReqParams = require('../utils/database-params'); function PgConnection (metadataBackend) { this.metadataBackend = metadataBackend; @@ -122,13 +123,7 @@ PgConnection.prototype.getConnection = function (username, callback) { if (err) { return callback(err); } - return callback(err, new PSQL({ - user: databaseParams.dbuser, - pass: databaseParams.dbpass, - host: databaseParams.dbhost, - port: databaseParams.dbport, - dbname: databaseParams.dbname - })); + return callback(err, new PSQL(dbParamsFromReqParams(databaseParams))); }); }; diff --git a/lib/backends/pg-query-runner.js b/lib/backends/pg-query-runner.js index 678e3057..f5236f54 100644 --- a/lib/backends/pg-query-runner.js +++ b/lib/backends/pg-query-runner.js @@ -1,6 +1,7 @@ 'use strict'; var PSQL = require('cartodb-psql'); +const dbParamsFromReqParams = require('../utils/database-params'); function PgQueryRunner (pgConnection) { this.pgConnection = pgConnection; @@ -21,13 +22,7 @@ PgQueryRunner.prototype.run = function (username, query, callback) { return callback(err); } - const psql = new PSQL({ - user: databaseParams.dbuser, - pass: databaseParams.dbpass, - host: databaseParams.dbhost, - port: databaseParams.dbport, - dbname: databaseParams.dbname - }); + const psql = new PSQL(dbParamsFromReqParams(databaseParams)); psql.query(query, function (err, resultSet) { resultSet = resultSet || {}; diff --git a/lib/models/dataview/overviews/aggregation.js b/lib/models/dataview/overviews/aggregation.js index 9c61aa3f..1ded6723 100644 --- a/lib/models/dataview/overviews/aggregation.js +++ b/lib/models/dataview/overviews/aggregation.js @@ -94,6 +94,8 @@ var CATEGORIES_LIMIT = 6; function Aggregation (query, options, queryRewriter, queryRewriteData, params, queries) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); + this._checkOptions(options); + this.query = query; this.queries = queries; this.column = options.column; @@ -219,7 +221,27 @@ var aggregationFnQueryTpl = { sum: dot.template('sum({{=it._aggregationColumn}}*_feature_count)') }; -Aggregation.prototype.getAggregationSql = function () { +const VALID_OPERATIONS = { + count: [], + sum: ['aggregationColumn'] +}; + +Aggregation.prototype._checkOptions = function (options) { + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error(`Aggregation does not support '${options.aggregation}' operation in dataview overview options`); + } + + const requiredOptions = VALID_OPERATIONS[options.aggregation]; + const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + + if (missingOptions.length > 0) { + throw new Error( + `Aggregation '${options.aggregation}' is missing some options for overview: ${missingOptions.join(',')}` + ); + } +}; + +Aggregation.prototype.getAggregationSql = function() { return aggregationFnQueryTpl[this.aggregation]({ _aggregationFn: this.aggregation, _aggregationColumn: this.aggregationColumn || 1 diff --git a/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js b/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js index d45d3c15..065ae7c4 100644 --- a/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js +++ b/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js @@ -89,7 +89,12 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC layers.push(layer); if (layersResult.datasource) { datasourceBuilder.withLayerDatasource(currentLayerIndex, { - user: dbAuth.dbuser + user: dbAuth.dbuser, + // Used internally (PSQL) + pass: dbAuth.dbpassword, + dbpassword: dbAuth.dbpassword, + // Used by Mapnik + password: dbAuth.dbpassword }); } currentLayerIndex++; diff --git a/lib/models/mapconfig/provider/named-map-provider.js b/lib/models/mapconfig/provider/named-map-provider.js index d10451a4..797f29ef 100644 --- a/lib/models/mapconfig/provider/named-map-provider.js +++ b/lib/models/mapconfig/provider/named-map-provider.js @@ -242,7 +242,7 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider { } dbParams.dbuser = databaseParams.dbuser; - dbParams.dbpass = databaseParams.dbpass; + dbParams.dbpassword = databaseParams.dbpassword; dbParams.dbhost = databaseParams.dbhost; dbParams.dbport = databaseParams.dbport; dbParams.dbname = databaseParams.dbname; diff --git a/lib/server-options.js b/lib/server-options.js index 8afa474c..e89c0b1c 100644 --- a/lib/server-options.js +++ b/lib/server-options.js @@ -31,11 +31,11 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, { }); rendererConfig.mapnik.queryRewriter = new OverviewsQueryRewriter({ - zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' + zoom_level: 'cartodb.CDB_ZoomFromScale(!scale_denominator!)' }); rendererConfig.mvt.queryRewriter = new OverviewsQueryRewriter({ - zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' + zoom_level: 'cartodb.CDB_ZoomFromScale(!scale_denominator!)' }); // Perform keyword substitution in statsd diff --git a/lib/utils/query-utils.js b/lib/utils/query-utils.js index 4919c14a..47de5686 100644 --- a/lib/utils/query-utils.js +++ b/lib/utils/query-utils.js @@ -7,8 +7,8 @@ module.exports.getQueryActualRowCount = function (query) { return `select COUNT(*) AS rows FROM (${substituteDummyTokens(query)}) AS __cdb_query`; }; -function getQueryRowEstimation (query) { - return 'select CDB_EstimateRowCount($windshaft$' + substituteDummyTokens(query) + '$windshaft$) as rows'; +function getQueryRowEstimation(query) { + return 'select cartodb.CDB_EstimateRowCount($windshaft$' + substituteDummyTokens(query) + '$windshaft$) as rows'; } module.exports.getQueryRowEstimation = getQueryRowEstimation; diff --git a/package-lock.json b/package-lock.json index c33c2ec8..e27deb70 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "8.0.0", + "version": "8.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -94,7 +94,7 @@ "resolved": "https://registry.npmjs.org/@carto/mapnik/-/mapnik-3.6.2-carto.16.tgz", "integrity": "sha512-RX8ov5EpEheToESVKiKnV5yMPLA2KxaX2ANAs9W4856oKFPdbGmB2buDz54mLhwBDfler9GVo0Bzr2ayRVLO2A==", "requires": { - "mapnik-vector-tile": "github:cartodb/mapnik-vector-tile#v1.6.1-carto.2", + "mapnik-vector-tile": "github:cartodb/mapnik-vector-tile#e7ca5471f9e5de81243e6035e70444321fc0a82f", "nan": "2.14.0", "node-pre-gyp": "0.13.0" } @@ -450,9 +450,9 @@ }, "dependencies": { "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.5.tgz", + "integrity": "sha512-J9dlskqUXK1OeTOYBEn5s8aMukWMwWfs+rPTn/jn50Ux4MNXVhubL1wu/j2t+H4NVI+cXEcCaYellqaPVGXNqQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -519,7 +519,7 @@ "integrity": "sha512-myLV2xo3q9oTT8m8M+c+UTD/ziDN7hrYtZ9yY00KvMnu2NsVeRQsTe8Yxq1GVS8vF9iYfcelwjVEGObPUdLtHw==", "requires": { "debug": "^3.1.0", - "pg": "github:cartodb/node-postgres#6.4.2-cdb2", + "pg": "github:cartodb/node-postgres#5417d7b29b7272ca2e71bb396899ab3f177a9ae6", "underscore": "~1.6.0" } }, @@ -599,9 +599,9 @@ "dev": true }, "chownr": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz", - "integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==" + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.3.tgz", + "integrity": "sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw==" }, "chroma-js": { "version": "1.1.1", @@ -779,11 +779,11 @@ "integrity": "sha512-vDPw+rDgn3bZe1+F/pyEwb1oMG2XTlRVgAa6B4KccTEpYgF8w6eQllVbQcfIJnZyvzFtFpxnpGtx8dd7DJp/Rw==" }, "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", "requires": { - "mimic-response": "^1.0.0" + "mimic-response": "^2.0.0" } }, "deep-eql": { @@ -1778,11 +1778,11 @@ "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-minipass": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.6.tgz", - "integrity": "sha512-crhvyXcMejjv3Z5d2Fa9sf5xLYVCF5O1c71QxbVnbLsmYMBEvDAftewesN/HhY03YRoA7zOMxjNGrF5svGaaeQ==", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", "requires": { - "minipass": "^2.2.1" + "minipass": "^2.6.0" } }, "fs.realpath": { @@ -2710,9 +2710,9 @@ "dev": true }, "graceful-fs": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz", - "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==" + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" }, "grainstore": { "version": "2.0.1", @@ -2722,7 +2722,7 @@ "carto": "0.16.3", "debug": "~3.1.0", "generic-pool": "~2.2.0", - "millstone": "github:cartodb/millstone#v0.6.17-carto.3", + "millstone": "github:cartodb/millstone#eeeb308fba4586343bb848fbf8ae0d180192627d", "postcss": "~5.2.8", "postcss-scss": "0.4.0", "postcss-strip-inline-comments": "0.1.5", @@ -2915,9 +2915,9 @@ "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" }, "hosted-git-info": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.4.tgz", - "integrity": "sha512-pzXIvANXEFrc5oFFXRMkbLPQ2rXRoDERwDLyrcUxGhaZhgP54BBSl9Oheh7Vv0T090cszWBxPjkQQ5Sq1PbBRQ==" + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.5.tgz", + "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==" }, "http-errors": { "version": "1.6.3", @@ -2960,9 +2960,9 @@ "dev": true }, "ignore-walk": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.2.tgz", - "integrity": "sha512-EXyErtpHbn75ZTsOADsfx6J/FPo6/5cjev46PXrcTpd8z3BoRkXgYu9/JVqrI7tusjmwCZutGeRJeU0Wo1e4Cw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", "requires": { "minimatch": "^3.0.4" } @@ -3526,9 +3526,9 @@ "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.0.0.tgz", + "integrity": "sha512-8ilDoEapqA4uQ3TwS0jakGONKXVJqpy+RpM+3b7pLdOjghCrEiGp9SRkFbUHAmZW9vdnrENWHjaweIoTIJExSQ==" }, "minimatch": { "version": "3.0.4", @@ -3544,9 +3544,9 @@ "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "minipass": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.6.0.tgz", - "integrity": "sha512-OuNZ0OHrrI+jswzmgivYBZ+fAAGHZA4293d5q0z631/I9QSw3yumKB92njxHIHiB1eAdGRsE+3CcOPkoEyV5FQ==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -3558,18 +3558,18 @@ "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" }, "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" } } }, "minizlib": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.2.tgz", - "integrity": "sha512-hR3At21uSrsjjDTWrbu0IMLTpnkpv8IIMFDFaoz43Tmu4LkmAXfH44vNNzpTnf+OAQQCHrb91y/wc2J4x5XgSQ==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", "requires": { - "minipass": "^2.2.1" + "minipass": "^2.9.0" } }, "mkdirp": { @@ -3743,9 +3743,9 @@ }, "dependencies": { "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.5.tgz", + "integrity": "sha512-J9dlskqUXK1OeTOYBEn5s8aMukWMwWfs+rPTn/jn50Ux4MNXVhubL1wu/j2t+H4NVI+cXEcCaYellqaPVGXNqQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -3796,9 +3796,9 @@ "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==" }, "npm-packlist": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.4.tgz", - "integrity": "sha512-zTLo8UcVYtDU3gdeaFu2Xu0n0EvelfHDGuqtNIn5RO7yQj4H1TqNdBc/yZjxnWA0PVB8D3Woyp0i5B43JwQ6Vw==", + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.6.tgz", + "integrity": "sha512-u65uQdb+qwtGvEJh/DgQgW1Xg7sqeNbmxYyrvlNznaVTjV3E5P6F/EFjM+BVHXl7JJlsdG8A64M0XI8FI/IOlg==", "requires": { "ignore-walk": "^3.0.1", "npm-bundled": "^1.0.1" @@ -4586,11 +4586,11 @@ "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" }, "simple-get": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.0.3.tgz", - "integrity": "sha512-Wvre/Jq5vgoz31Z9stYWPLn0PqRqmBDpFSdypAnHu5AvRVCYPRYGnvryNLiXu8GOBNDH82J2FRHUGMjjHUpXFw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", "requires": { - "decompress-response": "^3.3.0", + "decompress-response": "^4.2.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } @@ -4885,13 +4885,13 @@ } }, "tar": { - "version": "4.4.10", - "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz", - "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==", + "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", "requires": { "chownr": "^1.1.1", "fs-minipass": "^1.2.5", - "minipass": "^2.3.5", + "minipass": "^2.8.6", "minizlib": "^1.2.1", "mkdirp": "^0.5.0", "safe-buffer": "^5.1.2", @@ -4904,9 +4904,9 @@ "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" }, "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" } } }, @@ -4935,7 +4935,7 @@ "resolved": "https://registry.npmjs.org/torque.js/-/torque.js-3.1.1.tgz", "integrity": "sha512-kfIrmI7TGqJT/J9DH8Mgvd9VEwcvAtnvyYyqymSN6WZ5L4BaVQEQ+zu5FgLChNAqCaRkqGc7bKp0Hj9A0rempA==", "requires": { - "carto": "github:cartodb/carto#master", + "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", "d3": "3.5.17", "turbo-carto": "^0.21.1", "turf-jenks": "~1.0.1" @@ -5148,14 +5148,14 @@ "integrity": "sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU=" }, "windshaft": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-5.6.3.tgz", - "integrity": "sha512-ProHyEDICqIhTkNouT9elj5FP9DCtApEFo7SvPlZbkeYygn1OGLN89Rq0/2WuGK5Y6AbX8INa5iYGxdGYaWKqQ==", + "version": "5.6.4", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-5.6.4.tgz", + "integrity": "sha512-W+SbKM5CjpuPXMrLUXzZms6yDk0aQeIs3eAd75Ih6SYwiJo12/vxkHqMe+0KZnVj6JE/oe53tVfWHRibE70NJA==", "requires": { "@carto/cartonik": "^0.7.0", "@carto/mapnik": "3.6.2-carto.16", "canvas": "^2.4.1", - "carto": "github:cartodb/carto#0.15.1-cdb5", + "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", "cartodb-psql": "^0.14.0", "cartodb-query-tables": "^0.6.1", "debug": "3.1.0", @@ -5167,9 +5167,9 @@ }, "dependencies": { "cartodb-query-tables": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.6.1.tgz", - "integrity": "sha512-hQR9F5tQ6W6uGZk8Us/0fwkAsvYfbsHUzyKqBqDYue+jOa7FrGS+KpWokdLYHhR/ye3N3iR9RBTrIkwp6aoUww==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.6.3.tgz", + "integrity": "sha512-ijHl2Roh+0B1pP8SL3guEAu8tE6yNN3J/oxdUWCFOSKjHmXjwTzyJdjO+tONGcERmlWfS594SCFYElGIweSnQg==", "requires": { "decimal.js": "10.2.0" } @@ -5332,9 +5332,9 @@ }, "dependencies": { "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.5.tgz", + "integrity": "sha512-J9dlskqUXK1OeTOYBEn5s8aMukWMwWfs+rPTn/jn50Ux4MNXVhubL1wu/j2t+H4NVI+cXEcCaYellqaPVGXNqQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", diff --git a/package.json b/package.json index 4bbf7061..d3643a4d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "8.0.0", + "version": "8.0.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -49,7 +49,7 @@ "step-profiler": "0.3.0", "turbo-carto": "0.21.2", "underscore": "1.6.0", - "windshaft": "^5.6.3", + "windshaft": "5.6.4", "yargs": "11.1.0" }, "devDependencies": { diff --git a/test/acceptance/dataviews/overviews-test.js b/test/acceptance/dataviews/overviews-test.js index ea0e76b5..79f7531d 100644 --- a/test/acceptance/dataviews/overviews-test.js +++ b/test/acceptance/dataviews/overviews-test.js @@ -657,6 +657,176 @@ describe('dataviews using tables with overviews', function () { }); }); }); + + describe('agreggation validation', function (){ + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + function createMapConfig(options) { + return { + version: '1.8.0', + analyses: [ + { id: 'data-source', + type: 'source', + params: { + query: 'select * from test_table_overviews' + } + } + ], + dataviews: { + test_invalid_aggregation: { + type: 'aggregation', + source: {id: 'data-source'}, + options: options + } + }, + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { id: 'data-source' } + } + } + ] + }; + } + + it('should fail if missing column', function (done) { + var options = { + aggregation: "sum", + aggregationColumn: "value" + }; + var missingColumnMapConfig = createMapConfig(options); + + var testClient = new TestClient(missingColumnMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation expects 'column' in dataview options"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation expects 'column' in dataview options" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if no aggregation operation', function (done) { + var options = { + column: "value", + aggregationColumn: "value" + }; + var missingOperationMapConfig = createMapConfig(options); + + var testClient = new TestClient(missingOperationMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation expects 'aggregation' operation in dataview options"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation expects 'aggregation' operation in dataview options" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if fake operation', function (done) { + var options = { + column: "value", + aggregation: "wadus", + aggregationColumn: "value" + }; + var wrongOperationMapConfig = createMapConfig(options); + + var testClient = new TestClient(wrongOperationMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation does not support 'wadus' operation"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation does not support 'wadus' operation" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if invalid operation for overview', function (done) { + var options = { + column: "value", + aggregation: "avg", + aggregationColumn: "value" + }; + var wrongOperationMapConfig = createMapConfig(options); + + var testClient = new TestClient(wrongOperationMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation does not support 'avg' operation in dataview overview options"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation does not support 'avg' operation in dataview overview options" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if no aggregation column when needed', function (done) { + var options = { + column: "value", + aggregation: "sum" + }; + var missingOptionMapConfig = createMapConfig(options); + + var testClient = new TestClient(missingOptionMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation 'sum' is missing some options: aggregationColumn"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation 'sum' is missing some options: aggregationColumn" + }] + }); + + testClient.drain(done); + }); + }); + }); }); }); diff --git a/test/acceptance/ported/torque-boundaries-test.js b/test/acceptance/ported/torque-boundaries-test.js index 03b290ca..0cb1fa25 100644 --- a/test/acceptance/ported/torque-boundaries-test.js +++ b/test/acceptance/ported/torque-boundaries-test.js @@ -251,12 +251,12 @@ describe('torque boundary points', function () { assert.ok(!err, 'Failed to create layergroup'); var parsedBody = JSON.parse(res.body); - var expected_token = parsedBody.layergroupid; - layergroupIdToDelete = expected_token; + var expectedToken = parsedBody.layergroupid; + layergroupIdToDelete = expectedToken; var partialUrl = tileRequest.z + '/' + tileRequest.x + '/' + tileRequest.y; assert.response(server, { - url: '/api/v1/map/' + expected_token + '/0/' + partialUrl + '.json.torque', + url: '/api/v1/map/' + expectedToken + '/0/' + partialUrl + '.json.torque', method: 'GET', headers: { host: 'localhost' diff --git a/test/acceptance/ported/torque-test.js b/test/acceptance/ported/torque-test.js index 1c435146..feee249a 100644 --- a/test/acceptance/ported/torque-test.js +++ b/test/acceptance/ported/torque-test.js @@ -52,7 +52,7 @@ describe('torque', function () { }; step( - function do_post1 () { + function doPost1 () { var next = this; assert.response(server, { url: '/api/v1/map', @@ -71,7 +71,7 @@ describe('torque', function () { "Missing required property '-torque-frame-count' in torque layer CartoCSS"); return null; }, - function do_post2 (err) { + function doPost2 (err) { assert.ifError(err); var next = this; var css = 'Map { -torque-frame-count: 2; }'; @@ -93,7 +93,7 @@ describe('torque', function () { "Missing required property '-torque-resolution' in torque layer CartoCSS"); return null; }, - function do_post3 (err) { + function doPost3 (err) { assert.ifError(err); var next = this; var css = 'Map { -torque-frame-count: 2; -torque-resolution: 3; }'; @@ -139,7 +139,7 @@ describe('torque', function () { ] }; step( - function do_post1 () { + function doPost1 () { var next = this; assert.response(server, { url: '/api/v1/map', @@ -181,9 +181,9 @@ describe('torque', function () { ] }; - var expected_token; + var expectedToken; step( - function do_post () { + function doPost () { var next = this; assert.response(server, { url: '/api/v1/map', @@ -199,10 +199,10 @@ describe('torque', function () { // from layergroup creation via POST checkCORSHeaders(res); var parsedBody = JSON.parse(res.body); - if (expected_token) { - assert.deepStrictEqual(parsedBody, { layergroupid: expected_token, layercount: 2 }); + if (expectedToken) { + assert.deepStrictEqual(parsedBody, { layergroupid: expectedToken, layercount: 2 }); } else { - expected_token = parsedBody.layergroupid; + expectedToken = parsedBody.layergroupid; } var meta = parsedBody.metadata; assert.ok(!_.isUndefined(meta), @@ -220,11 +220,11 @@ describe('torque', function () { }); return null; }, - function do_get_tile (err) { + function doGetTile (err) { assert.ifError(err); var next = this; assert.response(server, { - url: '/api/v1/map/' + expected_token + '/0/0/0.png', + url: '/api/v1/map/' + expectedToken + '/0/0/0.png', method: 'GET', encoding: 'binary', headers: { @@ -232,7 +232,7 @@ describe('torque', function () { } }, {}, function (res, err) { next(err, res); }); }, - function check_mapnik_error_1 (err, res) { + function checkMapnikError1 (err, res) { assert.ifError(err); assert.strictEqual(res.statusCode, 400, res.statusCode + (res.statusCode !== 200 ? (': ' + res.body) : '')); var parsed = JSON.parse(res.body); @@ -240,18 +240,18 @@ describe('torque', function () { assert.strictEqual(parsed.errors[0], "No 'mapnik' layers in MapConfig"); return null; }, - function do_get_grid0 (err) { + function doGetGrid0 (err) { assert.ifError(err); var next = this; assert.response(server, { - url: '/api/v1/map/' + expected_token + '/0/0/0/0.grid.json', + url: '/api/v1/map/' + expectedToken + '/0/0/0/0.grid.json', method: 'GET', headers: { host: 'localhost' } }, {}, function (res, err) { next(err, res); }); }, - function check_mapnik_error_2 (err, res) { + function checkMapnikError2 (err, res) { assert.ifError(err); assert.strictEqual(res.statusCode, 400, res.statusCode + (res.statusCode !== 200 ? (': ' + res.body) : '')); var parsed = JSON.parse(res.body); @@ -259,48 +259,48 @@ describe('torque', function () { assert.strictEqual(parsed.errors[0], 'Unsupported format grid.json'); return null; }, - function do_get_torque0 (err) { + function doGetTorque0 (err) { assert.ifError(err); var next = this; assert.response(server, { - url: '/api/v1/map/' + expected_token + '/0/0/0/0.json.torque', + url: '/api/v1/map/' + expectedToken + '/0/0/0/0.json.torque', method: 'GET', headers: { host: 'localhost' } }, {}, function (res, err) { next(err, res); }); }, - function check_torque0_response (err, res) { + function checkTorque0Response (err, res) { assert.ifError(err); assert.strictEqual(res.statusCode, 200, res.body); assert.strictEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - var tile_content = [{ x__uint8: 43, y__uint8: 43, vals__uint8: [1, 1], dates__uint16: [0, 1] }]; + var tileContent = [{ x__uint8: 43, y__uint8: 43, vals__uint8: [1, 1], dates__uint16: [0, 1] }]; var parsed = JSON.parse(res.body); - assert.deepStrictEqual(tile_content, parsed); + assert.deepStrictEqual(tileContent, parsed); return null; }, - function do_get_torque0_1 (err) { + function doGetTorque01 (err) { assert.ifError(err); var next = this; assert.response(server, { - url: '/api/v1/map/' + expected_token + '/0/0/0/0.torque.json', + url: '/api/v1/map/' + expectedToken + '/0/0/0/0.torque.json', method: 'GET', headers: { host: 'localhost' } }, {}, function (res, err) { next(err, res); }); }, - function check_torque0_response_1 (err, res) { + function checkTorque0Response1 (err, res) { assert.ifError(err); assert.strictEqual(res.statusCode, 200, res.body); assert.strictEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - var tile_content = [{ x__uint8: 43, y__uint8: 43, vals__uint8: [1, 1], dates__uint16: [0, 1] }]; + var tileContent = [{ x__uint8: 43, y__uint8: 43, vals__uint8: [1, 1], dates__uint16: [0, 1] }]; var parsed = JSON.parse(res.body); - assert.deepStrictEqual(tile_content, parsed); + assert.deepStrictEqual(tileContent, parsed); return null; }, function finish (err) { - keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; + keysToDelete['map_cfg|' + LayergroupToken.parse(expectedToken).token] = 0; keysToDelete['user:localhost:mapviews:global'] = 5; done(err); } @@ -329,7 +329,7 @@ describe('torque', function () { ] }; step( - function do_post () { + function doPost () { var next = this; assert.response(server, { url: '/api/v1/map', @@ -377,7 +377,7 @@ describe('torque', function () { const defautlPort = global.environment.postgres.port; step( - function do_post () { + function doPost () { var next = this; global.environment.postgres.port = 54777; assert.response(server, { diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index d1bf16eb..e336ffce 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -18,14 +18,13 @@ SET default_with_oids = false; -- public user role DROP USER IF EXISTS :PUBLICUSER; CREATE USER :PUBLICUSER WITH PASSWORD ':PUBLICPASS'; -GRANT USAGE ON SCHEMA cartodb TO :PUBLICUSER; -GRANT ALL ON CDB_TableMetadata TO :PUBLICUSER; +SELECT current_setting('search_path') AS my_path \gset +ALTER ROLE :PUBLICUSER SET search_path = :my_path, cartodb; -- db owner role DROP USER IF EXISTS :TESTUSER; CREATE USER :TESTUSER WITH PASSWORD ':TESTPASS'; -GRANT USAGE ON SCHEMA cartodb TO :TESTUSER; -GRANT ALL ON CDB_TableMetadata TO :TESTUSER; +ALTER ROLE :TESTUSER SET search_path = :my_path, cartodb; -- regular user role 1 DROP USER IF EXISTS test_windshaft_regular1;