cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #916 from CartoDB/finalDetails

Browse Source 
Rate limits final details
This commit is contained in:
Simon Martín 2018-03-26 11:51:04 +02:00 committed by GitHub
commit 967a0c31fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 144 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/cartodb/middleware/rate-limit.js
View File

@ -39,12 +39,16 @@ function rateLimit(userLimitsApi, endpointGroup = null) {
res.set({ res.set({
'Carto-Rate-Limit-Limit': limit, 'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining, 'Carto-Rate-Limit-Remaining': remaining,
'Retry-After': retry,
'Carto-Rate-Limit-Reset': reset 'Carto-Rate-Limit-Reset': reset
}); });
if (isBlocked) { if (isBlocked) {
const rateLimitError = new Error('You are over the limits.'); // retry is floor rounded in seconds by redis-cell
res.set('Retry-After', retry + 1);
let rateLimitError = new Error(
'You are over platform\'s limits. Please contact us to know more details'
);
rateLimitError.http_status = 429; rateLimitError.http_status = 429;
rateLimitError.type = 'limit'; rateLimitError.type = 'limit';
rateLimitError.subtype = 'rate-limit'; rateLimitError.subtype = 'rate-limit';

6
lib/cartodb/middleware/vector-error.js
View File

@ -5,7 +5,7 @@ module.exports = function vectorError() {
return function vectorErrorMiddleware(err, req, res, next) { return function vectorErrorMiddleware(err, req, res, next) {
if(req.params.format === 'mvt') { if(req.params.format === 'mvt') {
if (isTimeoutError(err)) { if (isTimeoutError(err) || isRateLimitError(err)) {
res.set('Content-Type', 'application/x-protobuf'); res.set('Content-Type', 'application/x-protobuf');
return res.status(429).send(timeoutErrorVectorTile); return res.status(429).send(timeoutErrorVectorTile);
} }
@ -27,3 +27,7 @@ function isDatasourceTimeoutError (err) {
function isTimeoutError (err) { function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
} }
function isRateLimitError (err) {
return err.type === 'limit' && err.subtype === 'rate-limit';
}

164
test/acceptance/rate-limit.test.js
View File

@ -15,6 +15,7 @@ let redisClient;
let testClient; let testClient;
let keysToDelete = ['user:localhost:mapviews:global']; let keysToDelete = ['user:localhost:mapviews:global'];
const user = 'localhost'; const user = 'localhost';
let layergroupid;
const query = ` const query = `
SELECT SELECT
@ -68,13 +69,13 @@ const createMapConfig = ({
}); });
function setLimit(count, period, burst) { function setLimit(count, period, burst, endpoint = RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) {
redisClient.SELECT(8, err => { redisClient.SELECT(8, err => {
if (err) { if (err) {
return; return;
} }
const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`; const key = `limits:rate:store:${user}:maps:${endpoint}`;
redisClient.rpush(key, burst); redisClient.rpush(key, burst);
redisClient.rpush(key, count); redisClient.rpush(key, count);
redisClient.rpush(key, period); redisClient.rpush(key, period);
@ -87,8 +88,12 @@ function getReqAndRes() {
req: {}, req: {},
res: { res: {
headers: {}, headers: {},
set(headers) { set(headers, value) {
if(typeof headers === 'object') {
this.headers = headers; this.headers = headers;
} else {
this.headers[headers] = value;
}
}, },
locals: { locals: {
user: 'localhost' user: 'localhost'
@ -97,18 +102,21 @@ function getReqAndRes() {
}; };
} }
function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) { function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done) {
const response = { let response = {
status, status,
headers: { headers: {
'Content-Type': 'application/json; charset=utf-8', 'Content-Type': 'application/json; charset=utf-8',
'Carto-Rate-Limit-Limit': limit, 'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining, 'Carto-Rate-Limit-Remaining': remaining,
'Carto-Rate-Limit-Reset': reset, 'Carto-Rate-Limit-Reset': reset
'Retry-After': retry
} }
}; };
if(retry) {
response.headers['Retry-After'] = retry;
}
testClient.getLayergroup({ response }, err => { testClient.getLayergroup({ response }, err => {
assert.ifError(err); assert.ifError(err);
if (done) { if (done) {
@ -117,21 +125,26 @@ function assertGetLayergroupRequest (status, limit, remaining, reset, retry, don
}); });
} }
function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) { function assertRateLimitRequest (status, limit, remaining, reset, retry, done) {
const { req, res } = getReqAndRes(); const { req, res } = getReqAndRes();
rateLimit(req, res, function (err) { rateLimit(req, res, function (err) {
assert.deepEqual(res.headers, { let expectedHeaders = {
"Carto-Rate-Limit-Limit": limit, "Carto-Rate-Limit-Limit": limit,
"Carto-Rate-Limit-Remaining": remaining, "Carto-Rate-Limit-Remaining": remaining,
"Carto-Rate-Limit-Reset": reset, "Carto-Rate-Limit-Reset": reset
"Retry-After": retry };
});
if(retry) {
expectedHeaders['Retry-After'] = retry;
}
assert.deepEqual(res.headers, expectedHeaders);
if(status === 200) { if(status === 200) {
assert.ifError(err); assert.ifError(err);
} else { } else {
assert.ok(err); assert.ok(err);
assert.equal(err.message, 'You are over the limits.'); assert.equal(err.message, 'You are over platform\'s limits. Please contact us to know more details');
assert.equal(err.http_status, 429); assert.equal(err.http_status, 429);
assert.equal(err.type, 'limit'); assert.equal(err.type, 'limit');
assert.equal(err.subtype, 'rate-limit'); assert.equal(err.subtype, 'rate-limit');
@ -178,7 +191,7 @@ describe('rate limit', function() {
const burst = 1; const burst = 1;
setLimit(count, period, burst); setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1', done); assertGetLayergroupRequest(200, '2', '1', '1', null, done);
}); });
it("1 req/sec: 2 req/seg should be limited", function(done) { it("1 req/sec: 2 req/seg should be limited", function(done) {
@ -187,12 +200,12 @@ describe('rate limit', function() {
const burst = 1; const burst = 1;
setLimit(count, period, burst); setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1'); assertGetLayergroupRequest(200, '2', '1', '1');
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250); setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1'), 250);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500); setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 500);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750); setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 750);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950); setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 950);
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050); setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', null, done), 1050);
}); });
}); });
@ -234,12 +247,12 @@ describe('rate limit middleware', function () {
}); });
it("1 req/sec: 2 req/seg should be limited", function (done) { it("1 req/sec: 2 req/seg should be limited", function (done) {
assertRateLimitRequest(200, 1, 0, 1, -1); assertRateLimitRequest(200, 1, 0, 1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 250);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050);
}); });
it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) { it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) {
@ -248,13 +261,100 @@ describe('rate limit middleware', function () {
'SCRIPT', 'SCRIPT',
['FLUSH'], ['FLUSH'],
function () { function () {
assertRateLimitRequest(200, 1, 0, 1, -1); assertRateLimitRequest(200, 1, 0, 1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050);
} }
); );
}); });
}); });
describe('rate limit and vector tiles', function () {
before(function(done) {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true;
redisClient = redis.createClient(global.environment.redis.port);
const count = 1;
const period = 1;
const burst = 0;
setLimit(count, period, burst, RATE_LIMIT_ENDPOINTS_GROUPS.TILE);
testClient = new TestClient(createMapConfig(), 1234);
testClient.getLayergroup({status: 200}, (err, res) => {
assert.ifError(err);
layergroupid = res.layergroupid;
done();
});
});
after(function() {
global.environment.enabledFeatures.rateLimitsEnabled = false;
global.environment.enabledFeatures.rateLimitsByEndpoint.tile = false;
});
afterEach(function(done) {
keysToDelete.forEach( key => {
redisClient.del(key);
});
redisClient.SELECT(0, () => {
redisClient.del('user:localhost:mapviews:global');
redisClient.SELECT(5, () => {
redisClient.del('user:localhost:mapviews:global');
done();
});
});
});
it('mvt rate limited', function (done) {
const tileParams = (status, limit, remaining, reset, retry, contentType) => {
let headers = {
"Content-Type": contentType,
"Carto-Rate-Limit-Limit": limit,
"Carto-Rate-Limit-Remaining": remaining,
"Carto-Rate-Limit-Reset": reset
};
if (retry) {
headers['Retry-After'] = retry;
}
return {
layergroupid: layergroupid,
format: 'mvt',
response: {status, headers}
};
};
testClient.getTile(0, 0, 0, tileParams(204, '1', '0', '1'), (err) => {
assert.ifError(err);
testClient.getTile(
0,
0,
0,
tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'),
(err, res, tile) => {
assert.ifError(err);
var tileJSON = tile.toJSON();
assert.equal(Array.isArray(tileJSON), true);
assert.equal(tileJSON.length, 2);
assert.equal(tileJSON[0].name, 'errorTileSquareLayer');
assert.equal(tileJSON[1].name, 'errorTileStripesLayer');
done();
}
);
});
});
});