Merge pull request #1126 from CartoDB/gears

Be able to inject middlewares from configuration

NEWS.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 8.0.0
+Released 2019-mm-dd
+
+Breaking changes:
+- Schema change for "routes" in configuration file, each "router" is now an array instead of an object. See [`dd06de2`](https://github.com/CartoDB/Windshaft-cartodb/pull/1126/commits/dd06de2632661e19d64c9fbc2be0ba1a8059f54c) for more details.
+
+Announcements:
+
+- Added mechanism to inject custom middlewares through configuration.
+- Stop requiring unused config properties: "base_url", "base_url_mapconfig", and "base_url_templated".
+
 ## 7.2.0
 Released 2019-09-30
 

README.md

@@ -24,7 +24,7 @@ Create the config/environments/<env>.js files (there are .example files
 to start from). You can optionally use the ./configure script for this,
 see ```./configure --help``` to see available options.
 
-Look at lib/cartodb/server_options.js for more on config
+Look at lib/cartodb/server-options.js for more on config
 
 Upgrading
 ---------

app.js

@@ -6,7 +6,7 @@ var path = require('path');
 var fs = require('fs');
 var _ = require('underscore');
 var semver = require('semver');
-const setICUEnvVariable = require('./lib/cartodb/utils/icu_data_env_setter');
+const setICUEnvVariable = require('./lib/utils/icu-data-env-setter');
 
 // jshint undef:false
 var log = console.log.bind(console);
@@ -24,12 +24,12 @@ if (!semver.satisfies(nodejsVersion, engines.node)) {
 setICUEnvVariable();
 
 var argv = require('yargs')
-    .usage('Usage: $0 <environment> [options]')
+    .usage('Usage: node $0 <environment> [options]')
     .help('h')
     .example(
-    '$0 production -c /etc/sql-api/config.js',
-    'start server in production environment with /etc/sql-api/config.js as config file'
-)
+        'node $0 production -c /etc/windshaft-cartodb/config.js',
+        'start server in production environment with /etc/windshaft-cartodb/config.js as config file'
+    )
     .alias('h', 'help')
     .alias('c', 'config')
     .nargs('c', 1)
@@ -105,8 +105,8 @@ global.logger = global.log4js.getLogger();
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartodbWindshaft = require('./lib/cartodb/server');
-var serverOptions = require('./lib/cartodb/server_options');
+var cartodbWindshaft = require('./lib/server');
+var serverOptions = require('./lib/server-options');
 
 var server = cartodbWindshaft(serverOptions);
 

config/environments/development.js.example

@@ -16,47 +16,41 @@ var config = {
     // Base URLs for the APIs
     //
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    //
+    // Note: each entry corresponds with an express' router.
+    // You must define at least one path. However, middlewares are optional.
     ,routes: {
-        v1: {
+        api: [{
             paths: [
                 '/api/v1',
                 '/user/:user/api/v1',
             ],
-            // Base url for the Detached Maps API
-            // "/api/v1/map" is the new API,
-            map: {
-                paths: [
-                    '/map',
-                ]
-            },
-            // Base url for the Templated Maps API
-            // "/api/v1/map/named" is the new API,
-            template: {
-                paths: [
-                    '/map/named'
-                ]
-            }
-        },
-        // For compatibility with versions up to 1.6.x
-        v0: {
-            paths: [
-                '/tiles'
+            // Optional: attach middlewares at the begining of the router
+            // to perform custom operations.
+            middlewares: [
+                function noop () {
+                    return function noopMiddleware (req, res, next) {
+                        next();
+                    }
+                }
             ],
             // Base url for the Detached Maps API
-            // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
-            map: {
+            // "/api/v1/map" is the new API,
+            map: [{
                 paths: [
-                    '/layergroup'
-                ]
-            },
+                    '/map',
+                ],
+                middlewares: [] // Optional
+            }],
             // Base url for the Templated Maps API
-            // "/tiles/template" is for compatibility with versions up to 1.6.x
-            template: {
+            // "/api/v1/map/named" is the new API,
+            template: [{
                 paths: [
-                    '/template'
-                ]
-            }
-        }
+                    '/map/named'
+                ],
+                middlewares: [] // Optional
+            }]
+        }]
     }
 
     // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.

config/environments/production.js.example

@@ -16,47 +16,41 @@ var config = {
     // Base URLs for the APIs
     //
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    //
+    // Note: each entry corresponds with an express' router.
+    // You must define at least one path. However, middlewares are optional.
     ,routes: {
-        v1: {
+        api: [{
             paths: [
                 '/api/v1',
                 '/user/:user/api/v1',
             ],
-            // Base url for the Detached Maps API
-            // "/api/v1/map" is the new API,
-            map: {
-                paths: [
-                    '/map',
-                ]
-            },
-            // Base url for the Templated Maps API
-            // "/api/v1/map/named" is the new API,
-            template: {
-                paths: [
-                    '/map/named'
-                ]
-            }
-        },
-        // For compatibility with versions up to 1.6.x
-        v0: {
-            paths: [
-                '/tiles'
+            // Optional: attach middlewares at the begining of the router
+            // to perform custom operations.
+            middlewares: [
+                function noop () {
+                    return function noopMiddleware (req, res, next) {
+                        next();
+                    }
+                }
             ],
             // Base url for the Detached Maps API
-            // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
-            map: {
+            // "/api/v1/map" is the new API,
+            map: [{
                 paths: [
-                    '/layergroup'
-                ]
-            },
+                    '/map',
+                ],
+                middlewares: [] // Optional
+            }],
             // Base url for the Templated Maps API
-            // "/tiles/template" is for compatibility with versions up to 1.6.x
-            template: {
+            // "/api/v1/map/named" is the new API,
+            template: [{
                 paths: [
-                    '/template'
-                ]
-            }
-        }
+                    '/map/named'
+                ],
+                middlewares: [] // Optional
+            }]
+        }]
     }
 
     // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.

config/environments/staging.js.example

@@ -16,47 +16,41 @@ var config = {
     // Base URLs for the APIs
     //
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    //
+    // Note: each entry corresponds with an express' router.
+    // You must define at least one path. However, middlewares are optional.
     ,routes: {
-        v1: {
+        api: [{
             paths: [
                 '/api/v1',
                 '/user/:user/api/v1',
             ],
-            // Base url for the Detached Maps API
-            // "/api/v1/map" is the new API,
-            map: {
-                paths: [
-                    '/map',
-                ]
-            },
-            // Base url for the Templated Maps API
-            // "/api/v1/map/named" is the new API,
-            template: {
-                paths: [
-                    '/map/named'
-                ]
-            }
-        },
-        // For compatibility with versions up to 1.6.x
-        v0: {
-            paths: [
-                '/tiles'
+            // Optional: attach middlewares at the begining of the router
+            // to perform custom operations.
+            middlewares: [
+                function noop () {
+                    return function noopMiddleware (req, res, next) {
+                        next();
+                    }
+                }
             ],
             // Base url for the Detached Maps API
-            // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
-            map: {
+            // "/api/v1/map" is the new API,
+            map: [{
                 paths: [
-                    '/layergroup'
-                ]
-            },
+                    '/map',
+                ],
+                middlewares: [] // Optional
+            }],
             // Base url for the Templated Maps API
-            // "/tiles/template" is for compatibility with versions up to 1.6.x
-            template: {
+            // "/api/v1/map/named" is the new API,
+            template: [{
                 paths: [
-                    '/template'
-                ]
-            }
-        }
+                    '/map/named'
+                ],
+                middlewares: [] // Optional
+            }]
+        }]
     }
 
     // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.

config/environments/test.js.example

@@ -16,47 +16,41 @@ var config = {
     // Base URLs for the APIs
     //
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    //
+    // Note: each entry corresponds with an express' router.
+    // You must define at least one path. However, middlewares are optional.
     ,routes: {
-        v1: {
+        api: [{
             paths: [
                 '/api/v1',
                 '/user/:user/api/v1',
             ],
-            // Base url for the Detached Maps API
-            // "/api/v1/map" is the new API,
-            map: {
-                paths: [
-                    '/map',
-                ]
-            },
-            // Base url for the Templated Maps API
-            // "/api/v1/map/named" is the new API,
-            template: {
-                paths: [
-                    '/map/named'
-                ]
-            }
-        },
-        // For compatibility with versions up to 1.6.x
-        v0: {
-            paths: [
-                '/tiles'
+            // Optional: attach middlewares at the begining of the router
+            // to perform custom operations.
+            middlewares: [
+                function noop () {
+                    return function noopMiddleware (req, res, next) {
+                        next();
+                    }
+                }
             ],
             // Base url for the Detached Maps API
-            // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
-            map: {
+            // "/api/v1/map" is the new API,
+            map: [{
                 paths: [
-                    '/layergroup'
-                ]
-            },
+                    '/map',
+                ],
+                middlewares: [] // Optional
+            }],
             // Base url for the Templated Maps API
-            // "/tiles/template" is for compatibility with versions up to 1.6.x
-            template: {
+            // "/api/v1/map/named" is the new API,
+            template: [{
                 paths: [
-                    '/template'
-                ]
-            }
-        }
+                    '/map/named'
+                ],
+                middlewares: [] // Optional
+            }]
+        }]
     }
 
     // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.

lib/api/api-router.js

@@ -7,12 +7,12 @@ const cartodbRedis = require('cartodb-redis');
 
 const windshaft = require('windshaft');
 
-const PgConnection = require('../backends/pg_connection');
+const PgConnection = require('../backends/pg-connection');
 const AnalysisBackend = require('../backends/analysis');
 const AnalysisStatusBackend = require('../backends/analysis-status');
 const DataviewBackend = require('../backends/dataview');
-const TemplateMaps = require('../backends/template_maps.js');
-const PgQueryRunner = require('../backends/pg_query_runner');
+const TemplateMaps = require('../backends/template-maps');
+const PgQueryRunner = require('../backends/pg-query-runner');
 const StatsBackend = require('../backends/stats');
 const AuthBackend = require('../backends/auth');
 
@@ -23,12 +23,12 @@ const TablesExtentBackend = require('../backends/tables-extent');
 
 const ClusterBackend = require('../backends/cluster');
 
-const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables');
-const SurrogateKeysCache = require('../cache/surrogate_keys_cache');
-const VarnishHttpCacheBackend = require('../cache/backend/varnish_http');
+const LayergroupAffectedTablesCache = require('../cache/layergroup-affected-tables');
+const SurrogateKeysCache = require('../cache/surrogate-keys-cache');
+const VarnishHttpCacheBackend = require('../cache/backend/varnish-http');
 const FastlyCacheBackend = require('../cache/backend/fastly');
-const NamedMapProviderCache = require('../cache/named_map_provider_cache');
-const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+const NamedMapProviderCache = require('../cache/named-map-provider-cache');
+const NamedMapsCacheEntry = require('../cache/model/named-maps-entry');
 const NamedMapProviderReporter = require('../stats/reporter/named-map-provider');
 
 const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
@@ -196,16 +196,17 @@ module.exports = class ApiRouter {
         this.templateRouter = new TemplateRouter({ collaborators });
     }
 
-    register (app) {
+    route (app, routes) {
         // FIXME: we need a better way to reset cache while running tests
         if (process.env.NODE_ENV === 'test') {
             app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache;
         }
 
-        Object.keys(this.serverOptions.routes).forEach(apiVersion => {
-            const routes = this.serverOptions.routes[apiVersion];
-
+        routes.forEach(route => {
             const apiRouter = router({ mergeParams: true });
+            const { paths, middlewares = [] } = route;
+
+            middlewares.forEach(middleware => apiRouter.use(middleware()));
 
             apiRouter.use(logger(this.serverOptions));
             apiRouter.use(initializeStatusCode());
@@ -219,16 +220,14 @@ module.exports = class ApiRouter {
             apiRouter.use(cors());
             apiRouter.use(user());
 
-            this.templateRouter.register(apiRouter, routes.template.paths);
-            this.mapRouter.register(apiRouter, routes.map.paths);
+            this.templateRouter.route(apiRouter, route.template);
+            this.mapRouter.route(apiRouter, route.map);
 
             apiRouter.use(sendResponse());
             apiRouter.use(syntaxError());
             apiRouter.use(errorMiddleware());
 
-            const apiPaths = routes.paths;
-
-            apiPaths.forEach(path => app.use(path, apiRouter));
+            paths.forEach(path => app.use(path, apiRouter));
         });
     }
 };
@@ -283,7 +282,7 @@ function createSurrogateKeysCacheBackends(serverOptions) {
     return cacheBackends;
 }
 
-const timeoutErrorTilePath = __dirname + '/../../../assets/render-timeout-fallback.png';
+const timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
 const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
 function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) {

lib/api/map/analyses-catalog-controller.js

@@ -17,7 +17,7 @@ module.exports = class AnalysesController {
         this.userLimitsBackend = userLimitsBackend;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         mapRouter.get('/analyses/catalog', this.middlewares());
     }
 

lib/api/map/analysis-layergroup-controller.js

@@ -17,7 +17,7 @@ module.exports = class AnalysisLayergroupController {
         this.authBackend = authBackend;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         mapRouter.get('/:token/analysis/node/:nodeId', this.middlewares());
     }
 

lib/api/map/anonymous-map-controller.js

@@ -64,7 +64,7 @@ module.exports = class AnonymousMapController {
         this.layergroupMetadata = layergroupMetadata;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         mapRouter.options('/');
         mapRouter.get('/', this.middlewares());
         mapRouter.post('/', this.middlewares());

lib/api/map/attributes-layergroup-controller.js

@@ -32,7 +32,7 @@ module.exports = class AttributesLayergroupController {
         this.surrogateKeysCache = surrogateKeysCache;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         mapRouter.get('/:token/:layer/attributes/:fid', this.middlewares());
     }
 

lib/api/map/clustered-features-layergroup-controller.js

@@ -32,7 +32,7 @@ module.exports = class AggregatedFeaturesLayergroupController {
         this.surrogateKeysCache = surrogateKeysCache;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         mapRouter.get('/:token/:layer/:z/cluster/:clusterId', this.middlewares());
     }
 

lib/api/map/dataview-layergroup-controller.js

@@ -47,7 +47,7 @@ module.exports = class DataviewLayergroupController {
         this.surrogateKeysCache = surrogateKeysCache;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         // Undocumented/non-supported API endpoint methods.
         // Use at your own peril.
 

lib/api/map/map-router.js

@@ -126,19 +126,25 @@ module.exports = class MapRouter {
         );
     }
 
-    register (apiRouter, mapPaths) {
+    route (apiRouter, routes) {
         const mapRouter = router({ mergeParams: true });
 
-        this.analysisLayergroupController.register(mapRouter);
-        this.attributesLayergroupController.register(mapRouter);
-        this.dataviewLayergroupController.register(mapRouter);
-        this.previewLayergroupController.register(mapRouter);
-        this.tileLayergroupController.register(mapRouter);
-        this.anonymousMapController.register(mapRouter);
-        this.previewTemplateController.register(mapRouter);
-        this.analysesController.register(mapRouter);
-        this.clusteredFeaturesLayergroupController.register(mapRouter);
+        routes.forEach(route => {
+            const { paths, middlewares = [] } = route;
 
-        mapPaths.forEach(path => apiRouter.use(path, mapRouter));
+            middlewares.forEach(middleware => mapRouter.use(middleware()));
+
+            this.analysisLayergroupController.route(mapRouter);
+            this.attributesLayergroupController.route(mapRouter);
+            this.dataviewLayergroupController.route(mapRouter);
+            this.previewLayergroupController.route(mapRouter);
+            this.tileLayergroupController.route(mapRouter);
+            this.anonymousMapController.route(mapRouter);
+            this.previewTemplateController.route(mapRouter);
+            this.analysesController.route(mapRouter);
+            this.clusteredFeaturesLayergroupController.route(mapRouter);
+
+            paths.forEach(path => apiRouter.use(path, mapRouter));
+        });
     }
 };

lib/api/map/preview-layergroup-controller.js

@@ -35,7 +35,7 @@ module.exports = class PreviewLayergroupController {
         this.surrogateKeysCache = surrogateKeysCache;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         mapRouter.get('/static/center/:token/:z/:lat/:lng/:width/:height.:format', this.middlewares({
             validateZoom: true,
             previewType: 'centered'

lib/api/map/preview-template-controller.js

@@ -46,7 +46,7 @@ module.exports = class PreviewTemplateController {
         this.userLimitsBackend = userLimitsBackend;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         mapRouter.get('/static/named/:template_id/:width/:height.:format', this.middlewares());
     }
 

lib/api/map/tile-layergroup-controller.js

@@ -43,7 +43,7 @@ module.exports = class TileLayergroupController {
         this.surrogateKeysCache = surrogateKeysCache;
     }
 
-    register (mapRouter) {
+    route (mapRouter) {
         // REGEXP: doesn't match with `val`
         const not = (val) => `(?!${val})([^\/]+?)`;
 

lib/api/middlewares/stats.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const Profiler = require('../../stats/profiler_proxy');
+const Profiler = require('../../stats/profiler-proxy');
 const debug = require('debug')('windshaft:cartodb:stats');
 const onHeaders = require('on-headers');
 

lib/api/middlewares/surrogate-key-header.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const NamedMapsCacheEntry = require('../../cache/model/named_maps_entry');
+const NamedMapsCacheEntry = require('../../cache/model/named-maps-entry');
 const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider');
 
 module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {

lib/api/middlewares/user.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const CdbRequest = require('../../models/cdb_request');
+const CdbRequest = require('../../models/cdb-request');
 
 module.exports = function user () {
     const cdbRequest = new CdbRequest();

lib/api/middlewares/vector-error.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const fs = require('fs');
-const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../../assets/render-timeout-fallback.mvt');
+const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt');
 
 module.exports = function vectorError() {
     return function vectorErrorMiddleware(err, req, res, next) {

lib/api/template/admin-template-controller.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { templateName } = require('../../backends/template_maps');
+const { templateName } = require('../../backends/template-maps');
 const credentials = require('../middlewares/credentials');
 const rateLimit = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
@@ -18,7 +18,7 @@ module.exports = class AdminTemplateController {
         this.userLimitsBackend = userLimitsBackend;
     }
 
-    register (templateRouter) {
+    route (templateRouter) {
         templateRouter.options(`/:template_id`);
 
         templateRouter.post('/', this.middlewares({

lib/api/template/named-template-controller.js

@@ -63,7 +63,7 @@ module.exports = class NamedMapController {
         this.layergroupMetadata = layergroupMetadata;
     }
 
-    register (templateRouter) {
+    route (templateRouter) {
         templateRouter.get('/:template_id/jsonp', this.middlewares());
         templateRouter.post('/:template_id', this.middlewares());
     }

lib/api/template/template-router.js

@@ -54,13 +54,19 @@ module.exports = class TemplateRouter {
         );
     }
 
-    register (apiRouter, templatePaths) {
+    route (apiRouter, routes) {
         const templateRouter = router({ mergeParams: true });
 
-        this.namedMapController.register(templateRouter);
-        this.tileTemplateController.register(templateRouter);
-        this.adminTemplateController.register(templateRouter);
+        routes.forEach(route => {
+            const { paths, middlewares = [] } = route;
 
-        templatePaths.forEach(path => apiRouter.use(path, templateRouter));
+            middlewares.forEach(middleware => templateRouter.use(middleware()));
+
+            this.namedMapController.route(templateRouter);
+            this.tileTemplateController.route(templateRouter);
+            this.adminTemplateController.route(templateRouter);
+
+            paths.forEach(path => apiRouter.use(path, templateRouter));
+        });
     }
 };

lib/api/template/tile-template-controller.js

@@ -31,7 +31,7 @@ module.exports = class TileTemplateController {
         this.userLimitsBackend = userLimitsBackend;
     }
 
-    register (templateRouter) {
+    route (templateRouter) {
         templateRouter.get('/:template_id/:layer/:z/:x/:y.(:format)', this.middlewares());
     }
 

lib/backends/dataview.js

@@ -6,7 +6,7 @@ var BBoxFilter = require('../models/filter/bbox');
 var DataviewFactory = require('../models/dataview/factory');
 var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
 const dbParamsFromReqParams = require('../utils/database-params');
-var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter');
+var OverviewsQueryRewriter = require('../utils/overviews-query-rewriter');
 var overviewsQueryRewriter = new OverviewsQueryRewriter({
     zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
 });

lib/cache/named-map-provider-cache.js

@@ -3,7 +3,7 @@
 const LruCache = require('lru-cache');
 
 const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
-const { templateName } = require('../backends/template_maps');
+const { templateName } = require('../backends/template-maps');
 
 const TEN_MINUTES_IN_MILLISECONDS = 1000 * 60 * 10;
 const ACTIONS = ['update', 'delete'];

lib/models/dataview/factory.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const dataviews = require('./');
+const dataviews = require('.');
 
 module.exports = class DataviewFactory {
     static get dataviews() {

lib/models/dataview/overviews/factory.js

@@ -1,7 +1,7 @@
 'use strict';
 
 var parentFactory = require('../factory');
-var dataviews = require('./');
+var dataviews = require('.');
 
 function OverviewsDataviewFactory(queryRewriter, queryRewriteData, options) {
     this.queryRewriter = queryRewriter;