Merge branch 'spread-prepare-context-middleware' into unify-middlewares

.gitignore

@@ -11,3 +11,4 @@ redis.pid
 *.log
 coverage/
 .DS_Store
+libredis_cell.so

NEWS.md

@@ -1,13 +1,32 @@
 # Changelog
 
+## 6.0.0
+Released 2018-03-19
+Backward incompatible changes:
+ - Needs Redis v4
+
+New features:
+ - Upgrades camshaft to 0.61.8
+ - Upgrades cartodb-redis to 1.0.0
+ - Rate limit feature (disabled by default)
+
 ## 5.4.0
-Released yyyy-mm-dd
+Released 2018-03-15
  - Upgrades Windshaft to 4.5.7 ([Mapnik top metrics](https://github.com/CartoDB/Windshaft/pull/597), [AttributesBackend allows multiple features if all the attributes are the same](https://github.com/CartoDB/Windshaft/pull/602))
  - Implemented middleware to authorize users via new Api Key system
  - Keep the old authorization system as fallback
  - Aggregation widget: Remove NULL categories in 'count' aggregations too
  - Update request to 2.85.0
  - Update camshaft to 0.61.4 (Fixes for AOI and Merge analyses)
+ - Update windshaft to 4.6.0, which in turn updates @carto/mapnik to 3.6.2-carto.4 and related dependencies. It brings in a cache for rasterized symbols. See https://github.com/CartoDB/node-mapnik/blob/v3.6.2-carto/CHANGELOG.carto.md#362-carto4
+ - PostGIS: Variables in postgis SQL queries must now additionally be wrapped in `!` (refs [#29](https://github.com/CartoDB/mapnik/issues/29), [mapnik/#3618](https://github.com/mapnik/mapnik/pull/3618)):
+```sql
+-- Before
+SELECT ... WHERE trait = @variable
+
+-- Now
+SELECT ... WHERE trait = !@variable!
+```
 
 ## 5.3.1
 Released 2018-02-13

config/environments/development.js.example

@@ -343,7 +343,28 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerStats: true
+        layerStats: true,
+        // whether it should rate limit endpoints (global configuration)
+        rateLimitsEnabled: false,
+        // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
+        rateLimitsByEndpoint: {
+            anonymous: false,
+            static: false,
+            static_named: false,
+            dataview: false,
+            dataview_search: false,
+            analysis: false,
+            analysis_catalog: false,
+            tile: false,
+            attributes: false,
+            named_list: false,
+            named_create: false,
+            named_get: false,
+            named: false,
+            named_update: false,
+            named_delete: false,
+            named_tiles: false
+        }
     }
 };
 

config/environments/production.js.example

@@ -345,7 +345,28 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerStats: false
+        layerStats: false,
+        // whether it should rate limit endpoints (global configuration)
+        rateLimitsEnabled: false,
+        // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
+        rateLimitsByEndpoint: {
+            anonymous: false,
+            static: false,
+            static_named: false,
+            dataview: false,
+            dataview_search: false,
+            analysis: false,
+            analysis_catalog: false,
+            tile: false,
+            attributes: false,
+            named_list: false,
+            named_create: false,
+            named_get: false,
+            named: false,
+            named_update: false,
+            named_delete: false,
+            named_tiles: false
+        }
     }
 };
 

config/environments/staging.js.example

@@ -345,7 +345,28 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerStats: true
+        layerStats: true,
+        // whether it should rate limit endpoints (global configuration)
+        rateLimitsEnabled: false,
+        // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
+        rateLimitsByEndpoint: {
+            anonymous: false,
+            static: false,
+            static_named: false,
+            dataview: false,
+            dataview_search: false,
+            analysis: false,
+            analysis_catalog: false,
+            tile: false,
+            attributes: false,
+            named_list: false,
+            named_create: false,
+            named_get: false,
+            named: false,
+            named_update: false,
+            named_delete: false,
+            named_tiles: false
+        }
     }
 };
 

config/environments/test.js.example

@@ -339,7 +339,28 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerStats: true
+        layerStats: true,
+        // whether it should rate limit endpoints (global configuration)
+        rateLimitsEnabled: false,
+        // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
+        rateLimitsByEndpoint: {
+            anonymous: false,
+            static: false,
+            static_named: false,
+            dataview: false,
+            dataview_search: false,
+            analysis: false,
+            analysis_catalog: false,
+            tile: false,
+            attributes: false,
+            named_list: false,
+            named_create: false,
+            named_get: false,
+            named: false,
+            named_update: false,
+            named_delete: false,
+            named_tiles: false
+        }
     }
 };
 

lib/cartodb/api/user_limits_api.js

@@ -11,6 +11,8 @@ function UserLimitsApi(metadataBackend, options) {
     this.metadataBackend = metadataBackend;
     this.options = options || {};
     this.options.limits = this.options.limits || {};
+
+    this.preprareRateLimit();
 }
 
 module.exports = UserLimitsApi;
@@ -77,3 +79,13 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call
         callback
     );
 };
+
+UserLimitsApi.prototype.preprareRateLimit = function () {
+    if (this.options.limits.rateLimitsEnabled) {
+        this.metadataBackend.loadRateLimitsScript();
+    }
+};
+
+UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) {
+    this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback);
+};

lib/cartodb/controllers/analyses.js

@@ -7,11 +7,14 @@ const layergroupToken = require('../middleware/layergroup-token');
 const credentials = require('../middleware/credentials');
 const authorize = require('../middleware/authorize');
 const dbConnSetup = require('../middleware/db-conn-setup');
+const rateLimit = require('../middleware/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
 const sendResponse = require('../middleware/send-response');
 
-function AnalysesController(pgConnection, authApi) {
+function AnalysesController(pgConnection, authApi, userLimitsApi) {
     this.pgConnection = pgConnection;
     this.authApi = authApi;
+    this.userLimitsApi = userLimitsApi;
 }
 
 module.exports = AnalysesController;
@@ -25,6 +28,7 @@ AnalysesController.prototype.register = function (app) {
         cleanUpQueryParams(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),

lib/cartodb/controllers/layergroup.js

@@ -7,6 +7,8 @@ const layergroupToken = require('../middleware/layergroup-token');
 const credentials = require('../middleware/credentials');
 const dbConnSetup = require('../middleware/db-conn-setup');
 const authorize = require('../middleware/authorize');
+const rateLimit = require('../middleware/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
 const sendResponse = require('../middleware/send-response');
 const DataviewBackend = require('../backends/dataview');
 const AnalysisStatusBackend = require('../backends/analysis-status');
@@ -34,8 +36,18 @@ const SUPPORTED_FORMATS = {
  * @param {AnalysisBackend} analysisBackend
  * @constructor
  */
-function LayergroupController(pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
+function LayergroupController(
-                              surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend, authApi) {
+    pgConnection,
+    mapStore,
+    tileBackend,
+    previewBackend,
+    attributesBackend,
+    surrogateKeysCache,
+    userLimitsApi,
+    layergroupAffectedTables,
+    analysisBackend,
+    authApi
+) {
     this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
@@ -61,6 +73,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -85,6 +98,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -110,6 +124,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -134,6 +149,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -156,6 +172,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(['layer']),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -176,6 +193,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(['layer']),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -214,6 +232,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(allowedDataviewQueryParams),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -234,6 +253,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(allowedDataviewQueryParams),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -254,6 +274,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(allowedDataviewQueryParams),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -274,6 +295,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(allowedDataviewQueryParams),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -294,6 +316,7 @@ LayergroupController.prototype.register = function(app) {
         cleanUpQueryParams(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),

lib/cartodb/controllers/map.js

@@ -17,6 +17,8 @@ const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
 const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
 const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
 const LayergroupMetadata = require('../utils/layergroup-metadata');
+const rateLimit = require('../middleware/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
 
 /**
  * @param {AuthApi} authApi
@@ -31,9 +33,18 @@ const LayergroupMetadata = require('../utils/layergroup-metadata');
  * @param {StatsBackend} statsBackend
  * @constructor
  */
-function MapController(pgConnection, templateMaps, mapBackend, metadataBackend,
+function MapController (
-                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
+    pgConnection,
-                       statsBackend, authApi) {
+    templateMaps,
+    mapBackend,
+    metadataBackend,
+    surrogateKeysCache,
+    userLimitsApi,
+    layergroupAffectedTables,
+    mapConfigAdapter,
+    statsBackend,
+    authApi
+) {
     this.pgConnection = pgConnection;
     this.templateMaps = templateMaps;
     this.mapBackend = mapBackend;
@@ -55,18 +66,32 @@ module.exports = MapController;
 MapController.prototype.register = function(app) {
     const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app;
 
-    app.get(`${mapconfigBasePath}`, this.composeCreateMapMiddleware());
+    app.get(
-    app.post(`${mapconfigBasePath}`, this.composeCreateMapMiddleware());
+        `${mapconfigBasePath}`,
+        this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
+    );
+
+    app.post(
+        `${mapconfigBasePath}`,
+        this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
+    );
 
     const useTemplate = true;
 
-    app.get(`${templateBasePath}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate));
+    app.get(
-    app.post(`${templateBasePath}/:template_id`, this.composeCreateMapMiddleware(useTemplate));
+        `${templateBasePath}/:template_id/jsonp`,
+        this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
+    );
+
+    app.post(
+        `${templateBasePath}/:template_id`,
+        this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
+    );
 
     app.options(app.base_url_mapconfig, cors('Content-Type'));
 };
 
-MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) {
+MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) {
     const isTemplateInstantiation = useTemplate;
     const useTemplateHash = useTemplate;
     const includeQuery = !useTemplate;
@@ -78,6 +103,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal
         cleanUpQueryParams(['aggregation']),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, endpointGroup),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),

lib/cartodb/controllers/named_maps.js

@@ -9,6 +9,8 @@ const dbConnSetup = require('../middleware/db-conn-setup');
 const authorize = require('../middleware/authorize');
 const sendResponse = require('../middleware/send-response');
 const vectorError = require('../middleware/vector-error');
+const rateLimit = require('../middleware/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
 
 const DEFAULT_ZOOM_CENTER = {
     zoom: 1,
@@ -33,8 +35,17 @@ function getRequestParams(locals) {
     return params;
 }
 
-function NamedMapsController(namedMapProviderCache, tileBackend, previewBackend,
+function NamedMapsController (
-                             surrogateKeysCache, tablesExtentApi, metadataBackend, pgConnection, authApi) {
+    namedMapProviderCache,
+    tileBackend,
+    previewBackend,
+    surrogateKeysCache,
+    tablesExtentApi,
+    metadataBackend,
+    pgConnection,
+    authApi,
+    userLimitsApi
+) {
     this.namedMapProviderCache = namedMapProviderCache;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
@@ -43,6 +54,7 @@ function NamedMapsController(namedMapProviderCache, tileBackend, previewBackend,
     this.metadataBackend = metadataBackend;
     this.pgConnection = pgConnection;
     this.authApi = authApi;
+    this.userLimitsApi = userLimitsApi;
 }
 
 module.exports = NamedMapsController;
@@ -56,6 +68,7 @@ NamedMapsController.prototype.register = function(app) {
         cleanUpQueryParams(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),
@@ -84,6 +97,7 @@ NamedMapsController.prototype.register = function(app) {
         cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED),
         layergroupToken(),
         credentials(),
         authorize(this.authApi),

lib/cartodb/controllers/named_maps_admin.js

@@ -3,6 +3,8 @@ const cors = require('../middleware/cors');
 const user = require('../middleware/user');
 const locals = require('../middleware/locals');
 const credentials = require('../middleware/credentials');
+const rateLimit = require('../middleware/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
 const sendResponse = require('../middleware/send-response');
 
 /**
@@ -11,9 +13,10 @@ const sendResponse = require('../middleware/send-response');
  * @param {TemplateMaps} templateMaps
  * @constructor
  */
-function NamedMapsAdminController(authApi, templateMaps) {
+function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) {
     this.authApi = authApi;
     this.templateMaps = templateMaps;
+    this.userLimitsApi = userLimitsApi;
 }
 
 module.exports = NamedMapsAdminController;
@@ -26,6 +29,7 @@ NamedMapsAdminController.prototype.register = function (app) {
         cors(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE),
         credentials(),
         checkContentType({ action: 'POST', label: 'POST TEMPLATE' }),
         authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }),
@@ -38,6 +42,7 @@ NamedMapsAdminController.prototype.register = function (app) {
         cors(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE),
         credentials(),
         checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }),
         authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }),
@@ -50,6 +55,7 @@ NamedMapsAdminController.prototype.register = function (app) {
         cors(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET),
         credentials(),
         authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }),
         retrieveTemplate({ templateMaps: this.templateMaps }),
@@ -61,6 +67,7 @@ NamedMapsAdminController.prototype.register = function (app) {
         cors(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE),
         credentials(),
         authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }),
         destroyTemplate({ templateMaps: this.templateMaps }),
@@ -72,6 +79,7 @@ NamedMapsAdminController.prototype.register = function (app) {
         cors(),
         locals(),
         user(),
+        rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST),
         credentials(),
         authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }),
         listTemplates({ templateMaps: this.templateMaps }),

lib/cartodb/middleware/rate-limit.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const RATE_LIMIT_ENDPOINTS_GROUPS = {
+    ANONYMOUS: 'anonymous',
+    STATIC: 'static',
+    STATIC_NAMED: 'static_named',
+    DATAVIEW: 'dataview',
+    DATAVIEW_SEARCH: 'dataview_search',
+    ANALYSIS: 'analysis',
+    ANALYSIS_CATALOG: 'analysis_catalog',
+    TILE: 'tile',
+    ATTRIBUTES: 'attributes',
+    NAMED_LIST: 'named_list',
+    NAMED_CREATE: 'named_create',
+    NAMED_GET: 'named_get',
+    NAMED: 'named',
+    NAMED_UPDATE: 'named_update',
+    NAMED_DELETE: 'named_delete',
+    NAMED_TILES: 'named_tiles'
+};
+
+function rateLimit(userLimitsApi, endpointGroup = null) {
+    if (!isRateLimitEnabled(endpointGroup)) {
+        return function rateLimitDisabledMiddleware(req, res, next) { next(); };
+    }
+
+    return function rateLimitMiddleware(req, res, next) {
+        userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) {
+            if (err) {
+                return next(err);
+            }
+
+            if (!userRateLimit) {
+                return next();
+            }
+
+            const [isBlocked, limit, remaining, retry, reset] = userRateLimit;
+
+            res.set({
+                'Carto-Rate-Limit-Limit': limit,
+                'Carto-Rate-Limit-Remaining': remaining,
+                'Retry-After': retry,
+                'Carto-Rate-Limit-Reset': reset
+            });
+
+            if (isBlocked) {
+                const rateLimitError = new Error('You are over the limits.');
+                rateLimitError.http_status = 429;
+                rateLimitError.type = 'limit';
+                rateLimitError.subtype = 'rate-limit';
+                return next(rateLimitError);
+            }
+
+            return next();
+        });
+    };
+}
+
+
+function isRateLimitEnabled(endpointGroup) {
+    return global.environment.enabledFeatures.rateLimitsEnabled &&
+        endpointGroup &&
+        global.environment.enabledFeatures.rateLimitsByEndpoint[endpointGroup];
+}
+
+module.exports = rateLimit;
+module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS;

lib/cartodb/server.js

@@ -74,7 +74,8 @@ module.exports = function(serverOptions) {
     var userLimitsApi = new UserLimitsApi(metadataBackend, {
         limits: {
             cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
-            render: serverOptions.renderer.mapnik.limits.render || 0
+            render: serverOptions.renderer.mapnik.limits.render || 0,
+            rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
         }
     });
 
@@ -251,12 +252,13 @@ module.exports = function(serverOptions) {
         tablesExtentApi,
         metadataBackend,
         pgConnection,
-        authApi
+        authApi,
+        userLimitsApi
     ).register(app);
 
-    new controller.NamedMapsAdmin(authApi, templateMaps).register(app);
+    new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app);
 
-    new controller.Analyses(pgConnection, authApi).register(app);
+    new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app);
 
     new controller.ServerInfo(versions).register(app);
 

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "5.3.2",
+  "version": "6.0.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -26,10 +26,10 @@
   "dependencies": {
     "basic-auth": "^2.0.0",
     "body-parser": "^1.18.2",
-    "camshaft": "0.61.4",
+    "camshaft": "0.61.8",
     "cartodb-psql": "0.10.2",
     "cartodb-query-tables": "0.3.0",
-    "cartodb-redis": "0.16.0",
+    "cartodb-redis": "1.0.0",
     "debug": "^3.1.0",
     "dot": "~1.0.2",
     "express": "~4.16.0",
@@ -48,7 +48,7 @@
     "step-profiler": "~0.3.0",
     "turbo-carto": "0.20.2",
     "underscore": "~1.6.0",
-    "windshaft": "4.5.7",
+    "windshaft": "4.6.0",
     "yargs": "~5.0.0"
   },
   "devDependencies": {

run_tests.sh

@@ -6,6 +6,7 @@ OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
 OPT_COVERAGE=no      # run tests with coverage
 OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files
+OPT_REDIS_CELL=yes   # download redis cell
 
 export PGAPPNAME=cartodb_tiler_tester
 
@@ -49,6 +50,17 @@ die() {
 	exit 1
 }
 
+get_redis_cell() {
+  if test x"$OPT_REDIS_CELL" = xyes; then
+    echo "Downloading redis-cell"
+    curl -L https://github.com/brandur/redis-cell/releases/download/v0.2.2/redis-cell-v0.2.2-x86_64-unknown-linux-gnu.tar.gz --output redis-cell.tar.gz > /dev/null 2>&1
+    tar xvzf redis-cell.tar.gz > /dev/null 2>&1
+    mv libredis_cell.so ${BASEDIR}/test/support/libredis_cell.so
+    rm redis-cell.tar.gz
+    rm libredis_cell.d
+  fi
+}
+
 trap 'cleanup_and_exit' 1 2 3 5 9 13
 
 while [ -n "$1" ]; do
@@ -88,6 +100,10 @@ while [ -n "$1" ]; do
                 OPT_CREATE_PGSQL=no
                 shift
                 continue
+        elif test "$1" = "--norediscell"; then
+                OPT_REDIS_CELL=no
+                shift
+                continue
         else
                 break
         fi
@@ -99,14 +115,16 @@ if [ -z "$1" ]; then
         echo " --nocreate   do not create the test environment on start" >&2
         echo " --nodrop     do not drop the test environment on exit" >&2
         echo " --with-coverage   use istanbul to determine code coverage" >&2
+        echo " --norediscell  do not download redis-cell" >&2
         exit 1
 fi
 
 TESTS=$@
 
 if test x"$OPT_CREATE_REDIS" = xyes; then
+  get_redis_cell
   echo "Starting redis on port ${REDIS_PORT}"
-  echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log &
+  echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${BASEDIR}/test/support/libredis_cell.so > ${BASEDIR}/test.log &
   PID_REDIS=$!
   echo ${PID_REDIS} > ${BASEDIR}/redis.pid
 fi

test/acceptance/named_layers.js

@@ -108,7 +108,7 @@ describe('named_layers', function() {
     });
 
     beforeEach(function(done) {
-        global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true};
+        global.environment.enabledFeatures.cdbQueryTablesFromPostgres = true;
         templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) {
             if (err) {
                 return done(err);
@@ -125,7 +125,7 @@ describe('named_layers', function() {
     });
 
     afterEach(function(done) {
-        global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false};
+        global.environment.enabledFeatures.cdbQueryTablesFromPostgres = false;
         templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) {
             if (err) {
                 return done(err);

test/acceptance/rate-limit.test.js

@@ -0,0 +1,260 @@
+require('../support/test_helper');
+
+const assert = require('../support/assert');
+const redis = require('redis');
+const RedisPool = require('redis-mpool');
+const cartodbRedis = require('cartodb-redis');
+const TestClient = require('../support/test-client');
+const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api');
+const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware;
+
+let userLimitsApi; 
+let rateLimit;
+let redisClient;
+let testClient;
+let keysToDelete = ['user:localhost:mapviews:global'];
+const user = 'localhost';
+
+const query = `
+    SELECT
+        ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
+        1 cartodb_id,
+        2 val
+`;
+
+const createMapConfig = ({
+    version = '1.6.0',
+    type = 'cartodb',
+    sql = query,
+    cartocss = TestClient.CARTOCSS.POINTS,
+    cartocss_version = '2.3.0',
+    interactivity = 'cartodb_id',
+    countBy = 'cartodb_id'
+} = {}) => ({
+    version,
+    layers: [{
+        type,
+        options: {
+            source: {
+                id: 'a0'
+            },
+            cartocss,
+            cartocss_version,
+            interactivity
+        }
+    }],
+    analyses: [
+        {
+            id: 'a0',
+            type: 'source',
+            params: {
+                query: sql
+            }
+        }
+    ],
+    dataviews: {
+        count: {
+            source: {
+                id: 'a0'
+            },
+            type: 'formula',
+            options: {
+                column: countBy,
+                operation: 'count'
+            }
+        }
+    }
+});
+
+
+function setLimit(count, period, burst) {
+    redisClient.SELECT(8, err => {
+        if (err) {
+            return;
+        }
+
+        const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`;
+        redisClient.rpush(key, burst);
+        redisClient.rpush(key, count);
+        redisClient.rpush(key, period);
+        keysToDelete.push(key);
+    });
+}
+
+function getReqAndRes() {
+    return {
+        req: {},
+        res: {
+            headers: {},
+            set(headers) {
+                this.headers = headers;
+            },
+            locals: {
+                user: 'localhost'
+            }
+        }
+    };
+}
+
+function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) {
+    const response = {
+        status,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8',
+            'Carto-Rate-Limit-Limit': limit,
+            'Carto-Rate-Limit-Remaining': remaining,
+            'Carto-Rate-Limit-Reset': reset,
+            'Retry-After': retry
+        }
+    };
+    
+    testClient.getLayergroup({ response }, err => {
+        assert.ifError(err);
+        if (done) {
+            setTimeout(done, 1000);
+        }
+    });
+}
+
+function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) {
+    const { req, res } = getReqAndRes();
+    rateLimit(req, res, function (err) {
+        assert.deepEqual(res.headers, {
+            "Carto-Rate-Limit-Limit": limit,
+            "Carto-Rate-Limit-Remaining": remaining,
+            "Carto-Rate-Limit-Reset": reset,
+            "Retry-After": retry
+        });
+
+        if(status === 200) {
+            assert.ifError(err);
+        } else {
+            assert.ok(err);
+            assert.equal(err.message, 'You are over the limits.');
+            assert.equal(err.http_status, 429);
+            assert.equal(err.type, 'limit');
+            assert.equal(err.subtype, 'rate-limit');
+        }
+
+        if (done) {
+            setTimeout(done, 1000);
+        }
+    });
+}
+
+describe('rate limit', function() {
+    before(function() {
+        global.environment.enabledFeatures.rateLimitsEnabled = true;
+        global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true;
+        
+        redisClient = redis.createClient(global.environment.redis.port);
+        testClient = new TestClient(createMapConfig(), 1234);
+    });
+
+    after(function() {
+        global.environment.enabledFeatures.rateLimitsEnabled = false;
+        global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false;
+    });
+
+    afterEach(function(done) {
+        keysToDelete.forEach( key => {
+            redisClient.del(key);
+        });
+
+        redisClient.SELECT(0, () => {
+            redisClient.del('user:localhost:mapviews:global');
+
+            redisClient.SELECT(5, () => {
+                redisClient.del('user:localhost:mapviews:global');
+                done();
+            });
+        });
+    }); 
+
+    it('should not be rate limited', function (done) {
+        const count = 1;
+        const period = 1;
+        const burst = 1;
+        setLimit(count, period, burst);
+
+        assertGetLayergroupRequest(200, '2', '1', '1', '-1', done);
+    });
+
+    it("1 req/sec: 2 req/seg should be limited", function(done) {
+        const count = 1;
+        const period = 1;
+        const burst = 1;
+        setLimit(count, period, burst);
+
+        assertGetLayergroupRequest(200, '2', '1', '1', '-1');
+        setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250);
+        setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'),  500);
+        setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'),  750);
+        setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'),  950);
+        setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050);
+    });
+
+});
+
+
+describe('rate limit middleware', function () {
+    before(function (done) {
+        global.environment.enabledFeatures.rateLimitsEnabled = true;
+        global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true;
+
+        const redisPool = new RedisPool(global.environment.redis);
+        const metadataBackend = cartodbRedis({ pool: redisPool });
+        userLimitsApi = new UserLimitsApi(metadataBackend, {
+            limits: {
+                rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
+            }
+        });
+        rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS);
+
+        redisClient = redis.createClient(global.environment.redis.port);
+        testClient = new TestClient(createMapConfig(), 1234);
+
+
+        const count = 1;
+        const period = 1;
+        const burst = 0;
+        setLimit(count, period, burst);
+
+        setTimeout(done, 1000);
+    });
+
+    after(function () {
+        global.environment.enabledFeatures.rateLimitsEnabled = false;
+        global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false;
+
+        keysToDelete.forEach(key => {
+            redisClient.del(key);
+        });
+    });
+
+    it("1 req/sec: 2 req/seg should be limited", function (done) {
+        assertRateLimitRequest(200, 1, 0, 1, -1);
+        setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250);
+        setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
+        setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
+        setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
+        setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
+    });
+
+    it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) {
+        userLimitsApi.metadataBackend.redisCmd(
+            8, 
+            'SCRIPT', 
+            ['FLUSH'], 
+            function () {
+                assertRateLimitRequest(200, 1, 0, 1, -1);
+                setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
+                setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
+                setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
+                setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
+                setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
+            }
+        );
+    });
+});

yarn.lock

@@ -2,20 +2,20 @@
 # yarn lockfile v1
 
 
-"@carto/mapnik@3.6.2-carto.2", "@carto/mapnik@~3.6.2-carto.0":
+"@carto/mapnik@3.6.2-carto.4":
-  version "3.6.2-carto.2"
+  version "3.6.2-carto.4"
-  resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.2.tgz#45a055fd2d39530a873ef9ce5a325baacc81c196"
+  resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.4.tgz#54042a5dbea293c54e1bd286b32277694c5dc2d2"
   dependencies:
     mapnik-vector-tile "1.5.0"
     nan "~2.7.0"
     node-pre-gyp "~0.6.30"
     protozero "1.5.1"
 
-"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb3":
+"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb4":
-  version "2.5.1-cdb3"
+  version "2.5.1-cdb4"
-  resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/e61c7752c033595a273dcd1d4b267252b174bd28"
+  resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3eb554e5109199f50f457cec72ee288cffa5d6b3"
   dependencies:
-    "@carto/mapnik" "3.6.2-carto.2"
+    "@carto/mapnik" "3.6.2-carto.4"
     "@mapbox/sphericalmercator" "~1.0.1"
     mapnik-pool "~0.1.3"
 
@@ -23,11 +23,11 @@
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2"
 
-abaculus@cartodb/abaculus#2.0.3-cdb2:
+abaculus@cartodb/abaculus#2.0.3-cdb5:
-  version "2.0.3-cdb2"
+  version "2.0.3-cdb5"
-  resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc"
+  resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/b899cbea04b3e6093aa3ef32331920acd5f839a1"
   dependencies:
-    "@carto/mapnik" "~3.6.2-carto.0"
+    "@carto/mapnik" "3.6.2-carto.4"
     d3-queue "^2.0.2"
     sphericalmercator "1.0.x"
 
@@ -238,9 +238,9 @@ camelcase@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
 
-camshaft@0.61.4:
+camshaft@0.61.8:
-  version "0.61.4"
+  version "0.61.8"
-  resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.4.tgz#eeab7e69a07d6fbc39ebc01ad19acf979de8fb2a"
+  resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.8.tgz#75669c6c14791a93433e79a8892298e88cb0fce2"
   dependencies:
     async "^1.5.2"
     bunyan "1.8.1"
@@ -301,9 +301,9 @@ cartodb-query-tables@0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756"
 
-cartodb-redis@0.16.0:
+cartodb-redis@1.0.0:
-  version "0.16.0"
+  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.16.0.tgz#969312fd329b24a76bf6e5a4dd961445f2eda734"
+  resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-1.0.0.tgz#83b4888ba7abb5d5895c8958b7e15cf4882602aa"
   dependencies:
     dot "~1.0.2"
     redis-mpool "^0.5.0"
@@ -2239,11 +2239,11 @@ through@2:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
 
-tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb7:
+tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb8:
-  version "0.6.18-cdb7"
+  version "0.6.18-cdb8"
-  resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/488d2acd65c89cc5382d996eabe8dc1f5051ce0f"
+  resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/9cb4546c8fdd34ced0a41dbf70e143475b4e2067"
   dependencies:
-    "@carto/mapnik" "3.6.2-carto.2"
+    "@carto/mapnik" "3.6.2-carto.4"
     generic-pool "~2.4.0"
     mime "~1.6.0"
     sphericalmercator "~1.0.4"
@@ -2400,13 +2400,13 @@ window-size@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
 
-windshaft@4.5.7:
+windshaft@4.6.0:
-  version "4.5.7"
+  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.7.tgz#1cb2950e529d0a7f8bb55d56f4f568152f05c27e"
+  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.6.0.tgz#d9394aff73c0aa761207ad2b0f12d1c23ac41244"
   dependencies:
-    "@carto/mapnik" "3.6.2-carto.2"
+    "@carto/mapnik" "3.6.2-carto.4"
-    "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb3
+    "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb4
-    abaculus cartodb/abaculus#2.0.3-cdb2
+    abaculus cartodb/abaculus#2.0.3-cdb5
     canvas cartodb/node-canvas#1.6.2-cdb2
     carto cartodb/carto#0.15.1-cdb3
     cartodb-psql "^0.10.1"
@@ -2420,7 +2420,7 @@ windshaft@4.5.7:
     sphericalmercator "1.0.4"
     step "~0.0.6"
     tilelive "5.12.2"
-    tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb7
+    tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb8
     torque.js "~2.11.0"
     underscore "~1.6.0"