diff --git a/.gitignore b/.gitignore index ddeef2ef..1000315b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ redis.pid *.log coverage/ .DS_Store +libredis_cell.so diff --git a/NEWS.md b/NEWS.md index 3ebb1502..583aa88d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,7 +3,8 @@ ## 5.4.1 Released yyyy-mm-dd - Upgrades camshaft to 0.61.8 - + - Upgrades cartodb-redis to 1.0.0 + - Rate limit feature (disabled by default) ## 5.4.0 Released 2018-03-15 diff --git a/config/environments/development.js.example b/config/environments/development.js.example index df2273c7..12de0912 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -343,7 +343,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 9f7436df..beb9f15c 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -345,7 +345,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: false + layerStats: false, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 66c5a88c..40d3bb8b 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -345,7 +345,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 0c98c4b6..4d2c2d27 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -339,7 +339,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 0f4f0f4c..caf9f151 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -11,6 +11,8 @@ function UserLimitsApi(metadataBackend, options) { this.metadataBackend = metadataBackend; this.options = options || {}; this.options.limits = this.options.limits || {}; + + this.preprareRateLimit(); } module.exports = UserLimitsApi; @@ -77,3 +79,13 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call callback ); }; + +UserLimitsApi.prototype.preprareRateLimit = function () { + if (this.options.limits.rateLimitsEnabled) { + this.metadataBackend.loadRateLimitsScript(); + } +}; + +UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) { + this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback); +}; diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 1a78208f..3ed0d5a4 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -1,9 +1,12 @@ var PSQL = require('cartodb-psql'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -function AnalysesController(prepareContext) { +function AnalysesController(prepareContext, userLimitsApi) { this.prepareContext = prepareContext; + this.userLimitsApi = userLimitsApi; } module.exports = AnalysesController; @@ -13,6 +16,7 @@ AnalysesController.prototype.register = function (app) { `${app.base_url_mapconfig}/analyses/catalog`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), this.prepareContext, createPGClient(), getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 2d0bf710..977e4bad 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -2,6 +2,8 @@ const cors = require('../middleware/cors'); const userMiddleware = require('../middleware/user'); const allowQueryParams = require('../middleware/allow-query-params'); const vectorError = require('../middleware/vector-error'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const DataviewBackend = require('../backends/dataview'); const AnalysisStatusBackend = require('../backends/analysis-status'); const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); @@ -28,8 +30,18 @@ const SUPPORTED_FORMATS = { * @param {AnalysisBackend} analysisBackend * @constructor */ -function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) { +function LayergroupController( + prepareContext, + pgConnection, + mapStore, + tileBackend, + previewBackend, + attributesBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTables, + analysisBackend +) { this.pgConnection = pgConnection; this.mapStore = mapStore; this.tileBackend = tileBackend; @@ -54,6 +66,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/:z/:x/:y@:scale_factor?x.:format`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), getTile(this.tileBackend, 'map_tile'), @@ -73,6 +86,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/:z/:x/:y.:format`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), getTile(this.tileBackend, 'map_tile'), @@ -93,6 +107,7 @@ LayergroupController.prototype.register = function(app) { distinguishLayergroupFromStaticRoute(), cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), getTile(this.tileBackend, 'maplayer_tile'), @@ -112,6 +127,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/:layer/attributes/:fid`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), getFeatureAttributes(this.attributesBackend), @@ -129,6 +145,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), allowQueryParams(['layer']), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), @@ -145,6 +162,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), allowQueryParams(['layer']), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), @@ -179,6 +197,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/dataview/:dataviewName`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), @@ -195,6 +214,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/:layer/widget/:dataviewName`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), @@ -211,6 +231,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/dataview/:dataviewName/search`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), @@ -227,6 +248,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/:layer/widget/:dataviewName/search`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), @@ -243,6 +265,7 @@ LayergroupController.prototype.register = function(app) { `${basePath}/:token/analysis/node/:nodeId`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), this.prepareContext, analysisNodeStatus(this.analysisStatusBackend), sendResponse() diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 87e8b6be..01cd2c5e 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -11,6 +11,8 @@ const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../utils/layergroup-metadata'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** * @param {AuthApi} authApi @@ -50,14 +52,29 @@ MapController.prototype.register = function(app) { const { base_url_mapconfig, base_url_templated } = app; const useTemplate = true; - app.get(base_url_mapconfig, this.composeCreateMapMiddleware()); - app.post(base_url_mapconfig, this.composeCreateMapMiddleware()); - app.get(`${base_url_templated}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate)); - app.post(`${base_url_templated}/:template_id`, this.composeCreateMapMiddleware(useTemplate)); - app.options(app.base_url_mapconfig, cors('Content-Type')); + app.get( + base_url_mapconfig, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) + ); + app.post( + base_url_mapconfig, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) + ); + app.get( + `${base_url_templated}/:template_id/jsonp`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) + ); + app.post( + `${base_url_templated}/:template_id`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) + ); + app.options( + app.base_url_mapconfig, + cors('Content-Type') + ); }; -MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) { +MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { const isTemplateInstantiation = useTemplate; const useTemplateHash = useTemplate; const includeQuery = !useTemplate; @@ -67,6 +84,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal return [ cors(), userMiddleware(), + rateLimit(this.userLimitsApi, endpointGroup), allowQueryParams(['aggregation']), this.prepareContext, initProfiler(isTemplateInstantiation), diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index cda71346..04b80ff4 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -3,6 +3,8 @@ const cors = require('../middleware/cors'); const userMiddleware = require('../middleware/user'); const allowQueryParams = require('../middleware/allow-query-params'); const vectorError = require('../middleware/vector-error'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const DEFAULT_ZOOM_CENTER = { zoom: 1, @@ -27,14 +29,23 @@ function getRequestParams(locals) { return params; } -function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend, - surrogateKeysCache, tablesExtentApi, metadataBackend) { +function NamedMapsController( + prepareContext, + namedMapProviderCache, + tileBackend, + previewBackend, + surrogateKeysCache, + tablesExtentApi, + metadataBackend, + userLimitsApi +) { this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; this.metadataBackend = metadataBackend; + this.userLimitsApi = userLimitsApi; this.prepareContext = prepareContext; } @@ -47,6 +58,7 @@ NamedMapsController.prototype.register = function(app) { `${base_url_templated}/:template_id/:layer/:z/:x/:y.(:format)`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), this.prepareContext, getNamedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, @@ -70,6 +82,7 @@ NamedMapsController.prototype.register = function(app) { `${base_url_mapconfig}/static/named/:template_id/:width/:height.:format`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), this.prepareContext, getNamedMapProvider({ diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index 986fef5c..c9ba652e 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -1,6 +1,8 @@ const { templateName } = require('../backends/template_maps'); const cors = require('../middleware/cors'); const userMiddleware = require('../middleware/user'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const localsMiddleware = require('../middleware/context/locals'); const credentialsMiddleware = require('../middleware/context/credentials'); @@ -10,9 +12,10 @@ const credentialsMiddleware = require('../middleware/context/credentials'); * @param {TemplateMaps} templateMaps * @constructor */ -function NamedMapsAdminController(authApi, templateMaps) { +function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) { this.authApi = authApi; this.templateMaps = templateMaps; + this.userLimitsApi = userLimitsApi; } module.exports = NamedMapsAdminController; @@ -24,6 +27,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), localsMiddleware(), credentialsMiddleware(), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), @@ -36,6 +40,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/:template_id`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), localsMiddleware(), credentialsMiddleware(), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), @@ -48,6 +53,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/:template_id`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), localsMiddleware(), credentialsMiddleware(), authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), @@ -59,6 +65,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/:template_id`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), localsMiddleware(), credentialsMiddleware(), authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), @@ -70,6 +77,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/`, cors(), userMiddleware(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), localsMiddleware(), credentialsMiddleware(), authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), diff --git a/lib/cartodb/middleware/rate-limit.js b/lib/cartodb/middleware/rate-limit.js new file mode 100644 index 00000000..9a3daa9b --- /dev/null +++ b/lib/cartodb/middleware/rate-limit.js @@ -0,0 +1,67 @@ +'use strict'; + +const RATE_LIMIT_ENDPOINTS_GROUPS = { + ANONYMOUS: 'anonymous', + STATIC: 'static', + STATIC_NAMED: 'static_named', + DATAVIEW: 'dataview', + DATAVIEW_SEARCH: 'dataview_search', + ANALYSIS: 'analysis', + ANALYSIS_CATALOG: 'analysis_catalog', + TILE: 'tile', + ATTRIBUTES: 'attributes', + NAMED_LIST: 'named_list', + NAMED_CREATE: 'named_create', + NAMED_GET: 'named_get', + NAMED: 'named', + NAMED_UPDATE: 'named_update', + NAMED_DELETE: 'named_delete', + NAMED_TILES: 'named_tiles' +}; + +function rateLimit(userLimitsApi, endpointGroup = null) { + if (!isRateLimitEnabled(endpointGroup)) { + return function rateLimitDisabledMiddleware(req, res, next) { next(); }; + } + + return function rateLimitMiddleware(req, res, next) { + userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) { + if (err) { + return next(err); + } + + if (!userRateLimit) { + return next(); + } + + const [isBlocked, limit, remaining, retry, reset] = userRateLimit; + + res.set({ + 'Carto-Rate-Limit-Limit': limit, + 'Carto-Rate-Limit-Remaining': remaining, + 'Retry-After': retry, + 'Carto-Rate-Limit-Reset': reset + }); + + if (isBlocked) { + const rateLimitError = new Error('You are over the limits.'); + rateLimitError.http_status = 429; + rateLimitError.type = 'limit'; + rateLimitError.subtype = 'rate-limit'; + return next(rateLimitError); + } + + return next(); + }); + }; +} + + +function isRateLimitEnabled(endpointGroup) { + return global.environment.enabledFeatures.rateLimitsEnabled && + endpointGroup && + global.environment.enabledFeatures.rateLimitsByEndpoint[endpointGroup]; +} + +module.exports = rateLimit; +module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 532d155b..27b0178f 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -76,7 +76,8 @@ module.exports = function(serverOptions) { var userLimitsApi = new UserLimitsApi(metadataBackend, { limits: { cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, - render: serverOptions.renderer.mapnik.limits.render || 0 + render: serverOptions.renderer.mapnik.limits.render || 0, + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled } }); @@ -256,12 +257,13 @@ module.exports = function(serverOptions) { previewBackend, surrogateKeysCache, tablesExtentApi, - metadataBackend + metadataBackend, + userLimitsApi ).register(app); - new controller.NamedMapsAdmin(authApi, templateMaps).register(app); + new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app); - new controller.Analyses(prepareContext).register(app); + new controller.Analyses(prepareContext, userLimitsApi).register(app); new controller.ServerInfo(versions).register(app); diff --git a/package.json b/package.json index 5a5e00c5..e5b29af0 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "camshaft": "0.61.8", "cartodb-psql": "0.10.2", "cartodb-query-tables": "0.3.0", - "cartodb-redis": "0.16.0", + "cartodb-redis": "1.0.0", "debug": "^3.1.0", "dot": "~1.0.2", "express": "~4.16.0", diff --git a/run_tests.sh b/run_tests.sh index d1c63614..8412a851 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -6,6 +6,7 @@ OPT_DROP_REDIS=yes # drop the redis test environment OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment OPT_COVERAGE=no # run tests with coverage OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files +OPT_REDIS_CELL=yes # download redis cell export PGAPPNAME=cartodb_tiler_tester @@ -49,6 +50,17 @@ die() { exit 1 } +get_redis_cell() { + if test x"$OPT_REDIS_CELL" = xyes; then + echo "Downloading redis-cell" + curl -L https://github.com/brandur/redis-cell/releases/download/v0.2.2/redis-cell-v0.2.2-x86_64-unknown-linux-gnu.tar.gz --output redis-cell.tar.gz > /dev/null 2>&1 + tar xvzf redis-cell.tar.gz > /dev/null 2>&1 + mv libredis_cell.so ${BASEDIR}/test/support/libredis_cell.so + rm redis-cell.tar.gz + rm libredis_cell.d + fi +} + trap 'cleanup_and_exit' 1 2 3 5 9 13 while [ -n "$1" ]; do @@ -88,6 +100,10 @@ while [ -n "$1" ]; do OPT_CREATE_PGSQL=no shift continue + elif test "$1" = "--norediscell"; then + OPT_REDIS_CELL=no + shift + continue else break fi @@ -99,14 +115,16 @@ if [ -z "$1" ]; then echo " --nocreate do not create the test environment on start" >&2 echo " --nodrop do not drop the test environment on exit" >&2 echo " --with-coverage use istanbul to determine code coverage" >&2 + echo " --norediscell do not download redis-cell" >&2 exit 1 fi TESTS=$@ if test x"$OPT_CREATE_REDIS" = xyes; then + get_redis_cell echo "Starting redis on port ${REDIS_PORT}" - echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log & + echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${BASEDIR}/test/support/libredis_cell.so > ${BASEDIR}/test.log & PID_REDIS=$! echo ${PID_REDIS} > ${BASEDIR}/redis.pid fi diff --git a/test/acceptance/named_layers.js b/test/acceptance/named_layers.js index 9c0a9966..5844c479 100644 --- a/test/acceptance/named_layers.js +++ b/test/acceptance/named_layers.js @@ -108,7 +108,7 @@ describe('named_layers', function() { }); beforeEach(function(done) { - global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true}; + global.environment.enabledFeatures.cdbQueryTablesFromPostgres = true; templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) { if (err) { return done(err); @@ -125,7 +125,7 @@ describe('named_layers', function() { }); afterEach(function(done) { - global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false}; + global.environment.enabledFeatures.cdbQueryTablesFromPostgres = false; templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) { if (err) { return done(err); diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js new file mode 100644 index 00000000..339e187e --- /dev/null +++ b/test/acceptance/rate-limit.test.js @@ -0,0 +1,260 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const redis = require('redis'); +const RedisPool = require('redis-mpool'); +const cartodbRedis = require('cartodb-redis'); +const TestClient = require('../support/test-client'); +const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api'); +const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; + +let userLimitsApi; +let rateLimit; +let redisClient; +let testClient; +let keysToDelete = ['user:localhost:mapviews:global']; +const user = 'localhost'; + +const query = ` + SELECT + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = query, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); + + +function setLimit(count, period, burst) { + redisClient.SELECT(8, err => { + if (err) { + return; + } + + const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`; + redisClient.rpush(key, burst); + redisClient.rpush(key, count); + redisClient.rpush(key, period); + keysToDelete.push(key); + }); +} + +function getReqAndRes() { + return { + req: {}, + res: { + headers: {}, + set(headers) { + this.headers = headers; + }, + locals: { + user: 'localhost' + } + } + }; +} + +function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) { + const response = { + status, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + 'Carto-Rate-Limit-Limit': limit, + 'Carto-Rate-Limit-Remaining': remaining, + 'Carto-Rate-Limit-Reset': reset, + 'Retry-After': retry + } + }; + + testClient.getLayergroup({ response }, err => { + assert.ifError(err); + if (done) { + setTimeout(done, 1000); + } + }); +} + +function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) { + const { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.deepEqual(res.headers, { + "Carto-Rate-Limit-Limit": limit, + "Carto-Rate-Limit-Remaining": remaining, + "Carto-Rate-Limit-Reset": reset, + "Retry-After": retry + }); + + if(status === 200) { + assert.ifError(err); + } else { + assert.ok(err); + assert.equal(err.message, 'You are over the limits.'); + assert.equal(err.http_status, 429); + assert.equal(err.type, 'limit'); + assert.equal(err.subtype, 'rate-limit'); + } + + if (done) { + setTimeout(done, 1000); + } + }); +} + +describe('rate limit', function() { + before(function() { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; + + redisClient = redis.createClient(global.environment.redis.port); + testClient = new TestClient(createMapConfig(), 1234); + }); + + after(function() { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false; + }); + + afterEach(function(done) { + keysToDelete.forEach( key => { + redisClient.del(key); + }); + + redisClient.SELECT(0, () => { + redisClient.del('user:localhost:mapviews:global'); + + redisClient.SELECT(5, () => { + redisClient.del('user:localhost:mapviews:global'); + done(); + }); + }); + }); + + it('should not be rate limited', function (done) { + const count = 1; + const period = 1; + const burst = 1; + setLimit(count, period, burst); + + assertGetLayergroupRequest(200, '2', '1', '1', '-1', done); + }); + + it("1 req/sec: 2 req/seg should be limited", function(done) { + const count = 1; + const period = 1; + const burst = 1; + setLimit(count, period, burst); + + assertGetLayergroupRequest(200, '2', '1', '1', '-1'); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050); + }); + +}); + + +describe('rate limit middleware', function () { + before(function (done) { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; + + const redisPool = new RedisPool(global.environment.redis); + const metadataBackend = cartodbRedis({ pool: redisPool }); + userLimitsApi = new UserLimitsApi(metadataBackend, { + limits: { + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled + } + }); + rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS); + + redisClient = redis.createClient(global.environment.redis.port); + testClient = new TestClient(createMapConfig(), 1234); + + + const count = 1; + const period = 1; + const burst = 0; + setLimit(count, period, burst); + + setTimeout(done, 1000); + }); + + after(function () { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false; + + keysToDelete.forEach(key => { + redisClient.del(key); + }); + }); + + it("1 req/sec: 2 req/seg should be limited", function (done) { + assertRateLimitRequest(200, 1, 0, 1, -1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); + }); + + it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) { + userLimitsApi.metadataBackend.redisCmd( + 8, + 'SCRIPT', + ['FLUSH'], + function () { + assertRateLimitRequest(200, 1, 0, 1, -1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); + } + ); + }); +}); diff --git a/yarn.lock b/yarn.lock index afbdfea3..405e89b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -301,9 +301,9 @@ cartodb-query-tables@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756" -cartodb-redis@0.16.0: - version "0.16.0" - resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.16.0.tgz#969312fd329b24a76bf6e5a4dd961445f2eda734" +cartodb-redis@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-1.0.0.tgz#83b4888ba7abb5d5895c8958b7e15cf4882602aa" dependencies: dot "~1.0.2" redis-mpool "^0.5.0"