Merge pull request #916 from CartoDB/finalDetails
Rate limits final details
This commit is contained in:
commit
967a0c31fd
@ -39,12 +39,16 @@ function rateLimit(userLimitsApi, endpointGroup = null) {
|
||||
res.set({
|
||||
'Carto-Rate-Limit-Limit': limit,
|
||||
'Carto-Rate-Limit-Remaining': remaining,
|
||||
'Retry-After': retry,
|
||||
'Carto-Rate-Limit-Reset': reset
|
||||
});
|
||||
|
||||
if (isBlocked) {
|
||||
const rateLimitError = new Error('You are over the limits.');
|
||||
// retry is floor rounded in seconds by redis-cell
|
||||
res.set('Retry-After', retry + 1);
|
||||
|
||||
let rateLimitError = new Error(
|
||||
'You are over platform\'s limits. Please contact us to know more details'
|
||||
);
|
||||
rateLimitError.http_status = 429;
|
||||
rateLimitError.type = 'limit';
|
||||
rateLimitError.subtype = 'rate-limit';
|
||||
|
@ -5,7 +5,7 @@ module.exports = function vectorError() {
|
||||
return function vectorErrorMiddleware(err, req, res, next) {
|
||||
if(req.params.format === 'mvt') {
|
||||
|
||||
if (isTimeoutError(err)) {
|
||||
if (isTimeoutError(err) || isRateLimitError(err)) {
|
||||
res.set('Content-Type', 'application/x-protobuf');
|
||||
return res.status(429).send(timeoutErrorVectorTile);
|
||||
}
|
||||
@ -27,3 +27,7 @@ function isDatasourceTimeoutError (err) {
|
||||
function isTimeoutError (err) {
|
||||
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
|
||||
}
|
||||
|
||||
function isRateLimitError (err) {
|
||||
return err.type === 'limit' && err.subtype === 'rate-limit';
|
||||
}
|
||||
|
@ -15,6 +15,7 @@ let redisClient;
|
||||
let testClient;
|
||||
let keysToDelete = ['user:localhost:mapviews:global'];
|
||||
const user = 'localhost';
|
||||
let layergroupid;
|
||||
|
||||
const query = `
|
||||
SELECT
|
||||
@ -68,13 +69,13 @@ const createMapConfig = ({
|
||||
});
|
||||
|
||||
|
||||
function setLimit(count, period, burst) {
|
||||
function setLimit(count, period, burst, endpoint = RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) {
|
||||
redisClient.SELECT(8, err => {
|
||||
if (err) {
|
||||
return;
|
||||
}
|
||||
|
||||
const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`;
|
||||
const key = `limits:rate:store:${user}:maps:${endpoint}`;
|
||||
redisClient.rpush(key, burst);
|
||||
redisClient.rpush(key, count);
|
||||
redisClient.rpush(key, period);
|
||||
@ -87,8 +88,12 @@ function getReqAndRes() {
|
||||
req: {},
|
||||
res: {
|
||||
headers: {},
|
||||
set(headers) {
|
||||
this.headers = headers;
|
||||
set(headers, value) {
|
||||
if(typeof headers === 'object') {
|
||||
this.headers = headers;
|
||||
} else {
|
||||
this.headers[headers] = value;
|
||||
}
|
||||
},
|
||||
locals: {
|
||||
user: 'localhost'
|
||||
@ -97,18 +102,21 @@ function getReqAndRes() {
|
||||
};
|
||||
}
|
||||
|
||||
function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) {
|
||||
const response = {
|
||||
function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done) {
|
||||
let response = {
|
||||
status,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Carto-Rate-Limit-Limit': limit,
|
||||
'Carto-Rate-Limit-Remaining': remaining,
|
||||
'Carto-Rate-Limit-Reset': reset,
|
||||
'Retry-After': retry
|
||||
'Carto-Rate-Limit-Reset': reset
|
||||
}
|
||||
};
|
||||
|
||||
if(retry) {
|
||||
response.headers['Retry-After'] = retry;
|
||||
}
|
||||
|
||||
testClient.getLayergroup({ response }, err => {
|
||||
assert.ifError(err);
|
||||
if (done) {
|
||||
@ -117,21 +125,26 @@ function assertGetLayergroupRequest (status, limit, remaining, reset, retry, don
|
||||
});
|
||||
}
|
||||
|
||||
function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) {
|
||||
function assertRateLimitRequest (status, limit, remaining, reset, retry, done) {
|
||||
const { req, res } = getReqAndRes();
|
||||
rateLimit(req, res, function (err) {
|
||||
assert.deepEqual(res.headers, {
|
||||
let expectedHeaders = {
|
||||
"Carto-Rate-Limit-Limit": limit,
|
||||
"Carto-Rate-Limit-Remaining": remaining,
|
||||
"Carto-Rate-Limit-Reset": reset,
|
||||
"Retry-After": retry
|
||||
});
|
||||
"Carto-Rate-Limit-Reset": reset
|
||||
};
|
||||
|
||||
if(retry) {
|
||||
expectedHeaders['Retry-After'] = retry;
|
||||
}
|
||||
|
||||
assert.deepEqual(res.headers, expectedHeaders);
|
||||
|
||||
if(status === 200) {
|
||||
assert.ifError(err);
|
||||
} else {
|
||||
assert.ok(err);
|
||||
assert.equal(err.message, 'You are over the limits.');
|
||||
assert.equal(err.message, 'You are over platform\'s limits. Please contact us to know more details');
|
||||
assert.equal(err.http_status, 429);
|
||||
assert.equal(err.type, 'limit');
|
||||
assert.equal(err.subtype, 'rate-limit');
|
||||
@ -178,7 +191,7 @@ describe('rate limit', function() {
|
||||
const burst = 1;
|
||||
setLimit(count, period, burst);
|
||||
|
||||
assertGetLayergroupRequest(200, '2', '1', '1', '-1', done);
|
||||
assertGetLayergroupRequest(200, '2', '1', '1', null, done);
|
||||
});
|
||||
|
||||
it("1 req/sec: 2 req/seg should be limited", function(done) {
|
||||
@ -187,12 +200,12 @@ describe('rate limit', function() {
|
||||
const burst = 1;
|
||||
setLimit(count, period, burst);
|
||||
|
||||
assertGetLayergroupRequest(200, '2', '1', '1', '-1');
|
||||
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250);
|
||||
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500);
|
||||
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750);
|
||||
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950);
|
||||
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050);
|
||||
assertGetLayergroupRequest(200, '2', '1', '1');
|
||||
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1'), 250);
|
||||
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 500);
|
||||
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 750);
|
||||
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 950);
|
||||
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', null, done), 1050);
|
||||
});
|
||||
|
||||
});
|
||||
@ -234,12 +247,12 @@ describe('rate limit middleware', function () {
|
||||
});
|
||||
|
||||
it("1 req/sec: 2 req/seg should be limited", function (done) {
|
||||
assertRateLimitRequest(200, 1, 0, 1, -1);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
|
||||
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
|
||||
assertRateLimitRequest(200, 1, 0, 1);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 250);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950);
|
||||
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050);
|
||||
});
|
||||
|
||||
it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) {
|
||||
@ -248,13 +261,100 @@ describe('rate limit middleware', function () {
|
||||
'SCRIPT',
|
||||
['FLUSH'],
|
||||
function () {
|
||||
assertRateLimitRequest(200, 1, 0, 1, -1);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
|
||||
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
|
||||
assertRateLimitRequest(200, 1, 0, 1);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750);
|
||||
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950);
|
||||
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050);
|
||||
}
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('rate limit and vector tiles', function () {
|
||||
|
||||
before(function(done) {
|
||||
global.environment.enabledFeatures.rateLimitsEnabled = true;
|
||||
global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true;
|
||||
|
||||
redisClient = redis.createClient(global.environment.redis.port);
|
||||
const count = 1;
|
||||
const period = 1;
|
||||
const burst = 0;
|
||||
setLimit(count, period, burst, RATE_LIMIT_ENDPOINTS_GROUPS.TILE);
|
||||
|
||||
testClient = new TestClient(createMapConfig(), 1234);
|
||||
testClient.getLayergroup({status: 200}, (err, res) => {
|
||||
assert.ifError(err);
|
||||
|
||||
layergroupid = res.layergroupid;
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
after(function() {
|
||||
global.environment.enabledFeatures.rateLimitsEnabled = false;
|
||||
global.environment.enabledFeatures.rateLimitsByEndpoint.tile = false;
|
||||
});
|
||||
|
||||
afterEach(function(done) {
|
||||
keysToDelete.forEach( key => {
|
||||
redisClient.del(key);
|
||||
});
|
||||
|
||||
redisClient.SELECT(0, () => {
|
||||
redisClient.del('user:localhost:mapviews:global');
|
||||
|
||||
redisClient.SELECT(5, () => {
|
||||
redisClient.del('user:localhost:mapviews:global');
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('mvt rate limited', function (done) {
|
||||
const tileParams = (status, limit, remaining, reset, retry, contentType) => {
|
||||
let headers = {
|
||||
"Content-Type": contentType,
|
||||
"Carto-Rate-Limit-Limit": limit,
|
||||
"Carto-Rate-Limit-Remaining": remaining,
|
||||
"Carto-Rate-Limit-Reset": reset
|
||||
};
|
||||
|
||||
if (retry) {
|
||||
headers['Retry-After'] = retry;
|
||||
}
|
||||
|
||||
return {
|
||||
layergroupid: layergroupid,
|
||||
format: 'mvt',
|
||||
response: {status, headers}
|
||||
};
|
||||
};
|
||||
|
||||
testClient.getTile(0, 0, 0, tileParams(204, '1', '0', '1'), (err) => {
|
||||
assert.ifError(err);
|
||||
|
||||
testClient.getTile(
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'),
|
||||
(err, res, tile) => {
|
||||
assert.ifError(err);
|
||||
|
||||
var tileJSON = tile.toJSON();
|
||||
assert.equal(Array.isArray(tileJSON), true);
|
||||
assert.equal(tileJSON.length, 2);
|
||||
assert.equal(tileJSON[0].name, 'errorTileSquareLayer');
|
||||
assert.equal(tileJSON[1].name, 'errorTileStripesLayer');
|
||||
|
||||
done();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
|
Loading…
Reference in New Issue
Block a user