cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #875 from CartoDB/rateLimits

Browse Source 
Rate limit
This commit is contained in:
Simon Martín 2018-03-19 15:36:26 +01:00 committed by GitHub
commit d5e985fde5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 539 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -11,3 +11,4 @@ redis.pid
*.log
coverage/
.DS_Store
libredis_cell.so

3
NEWS.md
View File

@ -3,7 +3,8 @@
## 5.4.1
Released yyyy-mm-dd
- Upgrades camshaft to 0.61.8
- Upgrades cartodb-redis to 1.0.0
- Rate limit feature (disabled by default)
## 5.4.0
Released 2018-03-15

23
config/environments/development.js.example
View File

@ -343,7 +343,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: true
layerStats: true,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

23
config/environments/production.js.example
View File

@ -345,7 +345,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: false
layerStats: false,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

23
config/environments/staging.js.example
View File

@ -345,7 +345,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: true
layerStats: true,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

23
config/environments/test.js.example
View File

@ -339,7 +339,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: true
layerStats: true,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

12
lib/cartodb/api/user_limits_api.js
View File

@ -11,6 +11,8 @@ function UserLimitsApi(metadataBackend, options) {
this.metadataBackend = metadataBackend;
this.options = options || {};
this.options.limits = this.options.limits || {};
this.preprareRateLimit();
}
module.exports = UserLimitsApi;
@ -77,3 +79,13 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call
callback
);
};
UserLimitsApi.prototype.preprareRateLimit = function () {
if (this.options.limits.rateLimitsEnabled) {
this.metadataBackend.loadRateLimitsScript();
}
};
UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) {
this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback);
};

6
lib/cartodb/controllers/analyses.js
View File

@ -1,9 +1,12 @@
var PSQL = require('cartodb-psql');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
function AnalysesController(prepareContext) {
function AnalysesController(prepareContext, userLimitsApi) {
this.prepareContext = prepareContext;
this.userLimitsApi = userLimitsApi;
}
module.exports = AnalysesController;
@ -13,6 +16,7 @@ AnalysesController.prototype.register = function (app) {
`${app.base_url_mapconfig}/analyses/catalog`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG),
this.prepareContext,
createPGClient(),
getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),

27
lib/cartodb/controllers/layergroup.js
View File

@ -2,6 +2,8 @@ const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const DataviewBackend = require('../backends/dataview');
const AnalysisStatusBackend = require('../backends/analysis-status');
const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
@ -28,8 +30,18 @@ const SUPPORTED_FORMATS = {
* @param {AnalysisBackend} analysisBackend
* @constructor
*/
function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
function LayergroupController(
prepareContext,
pgConnection,
mapStore,
tileBackend,
previewBackend,
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
analysisBackend
) {
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
@ -54,6 +66,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'map_tile'),
@ -73,6 +86,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/:z/:x/:y.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'map_tile'),
@ -93,6 +107,7 @@ LayergroupController.prototype.register = function(app) {
distinguishLayergroupFromStaticRoute(),
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'maplayer_tile'),
@ -112,6 +127,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/:layer/attributes/:fid`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getFeatureAttributes(this.attributesBackend),
@ -129,6 +145,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
allowQueryParams(['layer']),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
@ -145,6 +162,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
allowQueryParams(['layer']),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
@ -179,6 +197,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/dataview/:dataviewName`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
@ -195,6 +214,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/:layer/widget/:dataviewName`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
@ -211,6 +231,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/dataview/:dataviewName/search`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
@ -227,6 +248,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/:layer/widget/:dataviewName/search`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
@ -243,6 +265,7 @@ LayergroupController.prototype.register = function(app) {
`${basePath}/:token/analysis/node/:nodeId`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
this.prepareContext,
analysisNodeStatus(this.analysisStatusBackend),
sendResponse()

30
lib/cartodb/controllers/map.js
View File

@ -11,6 +11,8 @@ const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
const LayergroupMetadata = require('../utils/layergroup-metadata');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
/**
* @param {AuthApi} authApi
@ -50,14 +52,29 @@ MapController.prototype.register = function(app) {
const { base_url_mapconfig, base_url_templated } = app;
const useTemplate = true;
app.get(base_url_mapconfig, this.composeCreateMapMiddleware());
app.post(base_url_mapconfig, this.composeCreateMapMiddleware());
app.get(`${base_url_templated}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate));
app.post(`${base_url_templated}/:template_id`, this.composeCreateMapMiddleware(useTemplate));
app.options(app.base_url_mapconfig, cors('Content-Type'));
app.get(
base_url_mapconfig,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.post(
base_url_mapconfig,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.get(
`${base_url_templated}/:template_id/jsonp`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.post(
`${base_url_templated}/:template_id`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.options(
app.base_url_mapconfig,
cors('Content-Type')
);
};
MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) {
MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) {
const isTemplateInstantiation = useTemplate;
const useTemplateHash = useTemplate;
const includeQuery = !useTemplate;
@ -67,6 +84,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal
return [
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, endpointGroup),
allowQueryParams(['aggregation']),
this.prepareContext,
initProfiler(isTemplateInstantiation),

17
lib/cartodb/controllers/named_maps.js
View File

@ -3,6 +3,8 @@ const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const DEFAULT_ZOOM_CENTER = {
zoom: 1,
@ -27,14 +29,23 @@ function getRequestParams(locals) {
return params;
}
function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
function NamedMapsController(
prepareContext,
namedMapProviderCache,
tileBackend,
previewBackend,
surrogateKeysCache,
tablesExtentApi,
metadataBackend,
userLimitsApi
) {
this.namedMapProviderCache = namedMapProviderCache;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.tablesExtentApi = tablesExtentApi;
this.metadataBackend = metadataBackend;
this.userLimitsApi = userLimitsApi;
this.prepareContext = prepareContext;
}
@ -47,6 +58,7 @@ NamedMapsController.prototype.register = function(app) {
`${base_url_templated}/:template_id/:layer/:z/:x/:y.(:format)`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES),
this.prepareContext,
getNamedMapProvider({
namedMapProviderCache: this.namedMapProviderCache,
@ -70,6 +82,7 @@ NamedMapsController.prototype.register = function(app) {
`${base_url_mapconfig}/static/named/:template_id/:width/:height.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED),
allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
this.prepareContext,
getNamedMapProvider({

10
lib/cartodb/controllers/named_maps_admin.js
View File

@ -1,6 +1,8 @@
const { templateName } = require('../backends/template_maps');
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const localsMiddleware = require('../middleware/context/locals');
const credentialsMiddleware = require('../middleware/context/credentials');
@ -10,9 +12,10 @@ const credentialsMiddleware = require('../middleware/context/credentials');
* @param {TemplateMaps} templateMaps
* @constructor
*/
function NamedMapsAdminController(authApi, templateMaps) {
function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) {
this.authApi = authApi;
this.templateMaps = templateMaps;
this.userLimitsApi = userLimitsApi;
}
module.exports = NamedMapsAdminController;
@ -24,6 +27,7 @@ NamedMapsAdminController.prototype.register = function (app) {
`${base_url_templated}/`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE),
localsMiddleware(),
credentialsMiddleware(),
checkContentType({ action: 'POST', label: 'POST TEMPLATE' }),
@ -36,6 +40,7 @@ NamedMapsAdminController.prototype.register = function (app) {
`${base_url_templated}/:template_id`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE),
localsMiddleware(),
credentialsMiddleware(),
checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }),
@ -48,6 +53,7 @@ NamedMapsAdminController.prototype.register = function (app) {
`${base_url_templated}/:template_id`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET),
localsMiddleware(),
credentialsMiddleware(),
authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }),
@ -59,6 +65,7 @@ NamedMapsAdminController.prototype.register = function (app) {
`${base_url_templated}/:template_id`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE),
localsMiddleware(),
credentialsMiddleware(),
authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }),
@ -70,6 +77,7 @@ NamedMapsAdminController.prototype.register = function (app) {
`${base_url_templated}/`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST),
localsMiddleware(),
credentialsMiddleware(),
authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }),

67
lib/cartodb/middleware/rate-limit.js Normal file
View File

@ -0,0 +1,67 @@
'use strict';
const RATE_LIMIT_ENDPOINTS_GROUPS = {
ANONYMOUS: 'anonymous',
STATIC: 'static',
STATIC_NAMED: 'static_named',
DATAVIEW: 'dataview',
DATAVIEW_SEARCH: 'dataview_search',
ANALYSIS: 'analysis',
ANALYSIS_CATALOG: 'analysis_catalog',
TILE: 'tile',
ATTRIBUTES: 'attributes',
NAMED_LIST: 'named_list',
NAMED_CREATE: 'named_create',
NAMED_GET: 'named_get',
NAMED: 'named',
NAMED_UPDATE: 'named_update',
NAMED_DELETE: 'named_delete',
NAMED_TILES: 'named_tiles'
};
function rateLimit(userLimitsApi, endpointGroup = null) {
if (!isRateLimitEnabled(endpointGroup)) {
return function rateLimitDisabledMiddleware(req, res, next) { next(); };
}
return function rateLimitMiddleware(req, res, next) {
userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) {
if (err) {
return next(err);
}
if (!userRateLimit) {
return next();
}
const [isBlocked, limit, remaining, retry, reset] = userRateLimit;
res.set({
'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining,
'Retry-After': retry,
'Carto-Rate-Limit-Reset': reset
});
if (isBlocked) {
const rateLimitError = new Error('You are over the limits.');
rateLimitError.http_status = 429;
rateLimitError.type = 'limit';
rateLimitError.subtype = 'rate-limit';
return next(rateLimitError);
}
return next();
});
};
}
function isRateLimitEnabled(endpointGroup) {
return global.environment.enabledFeatures.rateLimitsEnabled &&
endpointGroup &&
global.environment.enabledFeatures.rateLimitsByEndpoint[endpointGroup];
}
module.exports = rateLimit;
module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS;

10
lib/cartodb/server.js
View File

@ -76,7 +76,8 @@ module.exports = function(serverOptions) {
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
render: serverOptions.renderer.mapnik.limits.render || 0
render: serverOptions.renderer.mapnik.limits.render || 0,
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
}
});
@ -256,12 +257,13 @@ module.exports = function(serverOptions) {
previewBackend,
surrogateKeysCache,
tablesExtentApi,
metadataBackend
metadataBackend,
userLimitsApi
).register(app);
new controller.NamedMapsAdmin(authApi, templateMaps).register(app);
new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app);
new controller.Analyses(prepareContext).register(app);
new controller.Analyses(prepareContext, userLimitsApi).register(app);
new controller.ServerInfo(versions).register(app);

2
package.json
View File

@ -29,7 +29,7 @@
"camshaft": "0.61.8",
"cartodb-psql": "0.10.2",
"cartodb-query-tables": "0.3.0",
"cartodb-redis": "0.16.0",
"cartodb-redis": "1.0.0",
"debug": "^3.1.0",
"dot": "~1.0.2",
"express": "~4.16.0",

20
run_tests.sh
View File

@ -6,6 +6,7 @@ OPT_DROP_REDIS=yes # drop the redis test environment
OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment
OPT_COVERAGE=no # run tests with coverage
OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files
OPT_REDIS_CELL=yes # download redis cell
export PGAPPNAME=cartodb_tiler_tester
@ -49,6 +50,17 @@ die() {
exit 1
}
get_redis_cell() {
if test x"$OPT_REDIS_CELL" = xyes; then
echo "Downloading redis-cell"
curl -L https://github.com/brandur/redis-cell/releases/download/v0.2.2/redis-cell-v0.2.2-x86_64-unknown-linux-gnu.tar.gz --output redis-cell.tar.gz > /dev/null 2>&1
tar xvzf redis-cell.tar.gz > /dev/null 2>&1
mv libredis_cell.so ${BASEDIR}/test/support/libredis_cell.so
rm redis-cell.tar.gz
rm libredis_cell.d
fi
}
trap 'cleanup_and_exit' 1 2 3 5 9 13
while [ -n "$1" ]; do
@ -88,6 +100,10 @@ while [ -n "$1" ]; do
OPT_CREATE_PGSQL=no
shift
continue
elif test "$1" = "--norediscell"; then
OPT_REDIS_CELL=no
shift
continue
else
break
fi
@ -99,14 +115,16 @@ if [ -z "$1" ]; then
echo " --nocreate do not create the test environment on start" >&2
echo " --nodrop do not drop the test environment on exit" >&2
echo " --with-coverage use istanbul to determine code coverage" >&2
echo " --norediscell do not download redis-cell" >&2
exit 1
fi
TESTS=$@
if test x"$OPT_CREATE_REDIS" = xyes; then
get_redis_cell
echo "Starting redis on port ${REDIS_PORT}"
echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log &
echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${BASEDIR}/test/support/libredis_cell.so > ${BASEDIR}/test.log &
PID_REDIS=$!
echo ${PID_REDIS} > ${BASEDIR}/redis.pid
fi

4
test/acceptance/named_layers.js
View File

@ -108,7 +108,7 @@ describe('named_layers', function() {
});
beforeEach(function(done) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true};
global.environment.enabledFeatures.cdbQueryTablesFromPostgres = true;
templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) {
if (err) {
return done(err);
@ -125,7 +125,7 @@ describe('named_layers', function() {
});
afterEach(function(done) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false};
global.environment.enabledFeatures.cdbQueryTablesFromPostgres = false;
templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) {
if (err) {
return done(err);

260
test/acceptance/rate-limit.test.js Normal file
View File

@ -0,0 +1,260 @@
require('../support/test_helper');
const assert = require('../support/assert');
const redis = require('redis');
const RedisPool = require('redis-mpool');
const cartodbRedis = require('cartodb-redis');
const TestClient = require('../support/test-client');
const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api');
const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware;
let userLimitsApi;
let rateLimit;
let redisClient;
let testClient;
let keysToDelete = ['user:localhost:mapviews:global'];
const user = 'localhost';
const query = `
SELECT
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = query,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id'
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
function setLimit(count, period, burst) {
redisClient.SELECT(8, err => {
if (err) {
return;
}
const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`;
redisClient.rpush(key, burst);
redisClient.rpush(key, count);
redisClient.rpush(key, period);
keysToDelete.push(key);
});
}
function getReqAndRes() {
return {
req: {},
res: {
headers: {},
set(headers) {
this.headers = headers;
},
locals: {
user: 'localhost'
}
}
};
}
function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) {
const response = {
status,
headers: {
'Content-Type': 'application/json; charset=utf-8',
'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining,
'Carto-Rate-Limit-Reset': reset,
'Retry-After': retry
}
};
testClient.getLayergroup({ response }, err => {
assert.ifError(err);
if (done) {
setTimeout(done, 1000);
}
});
}
function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) {
const { req, res } = getReqAndRes();
rateLimit(req, res, function (err) {
assert.deepEqual(res.headers, {
"Carto-Rate-Limit-Limit": limit,
"Carto-Rate-Limit-Remaining": remaining,
"Carto-Rate-Limit-Reset": reset,
"Retry-After": retry
});
if(status === 200) {
assert.ifError(err);
} else {
assert.ok(err);
assert.equal(err.message, 'You are over the limits.');
assert.equal(err.http_status, 429);
assert.equal(err.type, 'limit');
assert.equal(err.subtype, 'rate-limit');
}
if (done) {
setTimeout(done, 1000);
}
});
}
describe('rate limit', function() {
before(function() {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true;
redisClient = redis.createClient(global.environment.redis.port);
testClient = new TestClient(createMapConfig(), 1234);
});
after(function() {
global.environment.enabledFeatures.rateLimitsEnabled = false;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false;
});
afterEach(function(done) {
keysToDelete.forEach( key => {
redisClient.del(key);
});
redisClient.SELECT(0, () => {
redisClient.del('user:localhost:mapviews:global');
redisClient.SELECT(5, () => {
redisClient.del('user:localhost:mapviews:global');
done();
});
});
});
it('should not be rate limited', function (done) {
const count = 1;
const period = 1;
const burst = 1;
setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1', done);
});
it("1 req/sec: 2 req/seg should be limited", function(done) {
const count = 1;
const period = 1;
const burst = 1;
setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1');
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950);
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050);
});
});
describe('rate limit middleware', function () {
before(function (done) {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true;
const redisPool = new RedisPool(global.environment.redis);
const metadataBackend = cartodbRedis({ pool: redisPool });
userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
}
});
rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS);
redisClient = redis.createClient(global.environment.redis.port);
testClient = new TestClient(createMapConfig(), 1234);
const count = 1;
const period = 1;
const burst = 0;
setLimit(count, period, burst);
setTimeout(done, 1000);
});
after(function () {
global.environment.enabledFeatures.rateLimitsEnabled = false;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false;
keysToDelete.forEach(key => {
redisClient.del(key);
});
});
it("1 req/sec: 2 req/seg should be limited", function (done) {
assertRateLimitRequest(200, 1, 0, 1, -1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
});
it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) {
userLimitsApi.metadataBackend.redisCmd(
8,
'SCRIPT',
['FLUSH'],
function () {
assertRateLimitRequest(200, 1, 0, 1, -1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
}
);
});
});

6
yarn.lock
View File

@ -301,9 +301,9 @@ cartodb-query-tables@0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756"
cartodb-redis@0.16.0:
version "0.16.0"
resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.16.0.tgz#969312fd329b24a76bf6e5a4dd961445f2eda734"
cartodb-redis@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-1.0.0.tgz#83b4888ba7abb5d5895c8958b7e15cf4882602aa"
dependencies:
dot "~1.0.2"
redis-mpool "^0.5.0"