cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP: implement timeout limit for raster

Browse Source
This commit is contained in:
Daniel García Aubert 2017-07-18 20:50:31 +02:00
parent 669707b26c
commit 87eb5407a8
3 changed files with 120 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
test/acceptance/user-timeout-limit.js
View File

@ -1,48 +1,25 @@
require('../support/test_helper');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
var testHelper = require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
var redis = require('redis');
var keysToDelete;
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
function withUserTimeoutRenderLimit(redisClient, user, userTimeoutLimit, callback) {
redisClient.SELECT(5, function(err) {
if (err) {
return callback(err);
}
var pointSleepSql = `
SELECT
pg_sleep(0.5),
'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator,
1 cartodb_id
`;
var userTimeoutLimitsKey = 'limits:timeout:' + user;
var redisParams = [
userTimeoutLimitsKey,
'render', userTimeoutLimit,
'render_public', userTimeoutLimit
];
redisClient.hmset(redisParams, function (err) {
if (err) {
return callback(err);
}
keysToDelete[userTimeoutLimitsKey] = 5;
return callback();
});
});
}
function createMapConfig (cartocss) {
function createMapConfig (sql = pointSleepSql, cartocss = TestClient.CARTOCSS.POINTS) {
return {
version: '1.6.0',
layers: [{
type: "cartodb",
type: 'cartodb',
options: {
sql: [
'SELECT',
' pg_sleep(1),',
' 1 cartodb_id,',
' \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator'
].join('\n'),
cartocss: cartocss,
sql,
cartocss,
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
@ -51,29 +28,64 @@ function createMapConfig (cartocss) {
}
describe('user timeout limits', function () {
var redisClient = redis.createClient(global.environment.redis.port);
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function() {
keysToDelete = {};
before(function () {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
});
afterEach(function (done) {
testHelper.deleteRedisKeys(keysToDelete, done);
after(function () {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
});
it('layergroup creation works even if test tile is slow', function (done) {
withUserTimeoutRenderLimit(redisClient, 'localhost', 1, function (err) {
if (err) {
return done(err);
}
it('layergroup creation works if test tile is fast but tile request fails if they are slow', function (done) {
var testClient = new TestClient(createMapConfig(), 1234);
var mapConfig = createMapConfig(TestClient.CARTOCSS.POINTS);
var testClient = new TestClient(mapConfig, 1234);
testClient.getTile(4, 4, 4, {}, function (err /*, res, tile */) {
assert.ok(err, err);
// TODO: check timeout tile
testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) {
assert.ifError(err);
testClient.getTile(0, 0, 0, {}, function (err, res, tile) {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategy;
beforeEach(function() {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
});
afterEach(function() {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
});
it('layergroup creation works even if test tile is slow', function (done) {
var testClient = new TestClient(createMapConfig(), 1234);
testClient.setUserRenderTimeoutLimit('localhost', 50, function (err) {
assert.ifError(err);
var params = {
status: 400,
contentType: 'application/json; charset=utf-8'
};
testClient.getTile(0, 0, 0, params, function (err, res, tile) {
assert.ifError(err);
assert.equal(tile.errors[0], 'Render timed out');
testClient.drain(done);
});
});
});
});
});

55
test/support/test-client.js
View File

@ -14,13 +14,13 @@ var helper = require('./test_helper');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
serverOptions.analysis.batch.inlineExecution = true;
var server = new CartodbWindshaft(serverOptions);
function TestClient(config, apiKey) {
this.mapConfig = isMapConfig(config) ? config : null;
this.template = isTemplate(config) ? config : null;
this.apiKey = apiKey;
this.keysToDelete = {};
this.server = new CartodbWindshaft(serverOptions);
}
module.exports = TestClient;
@ -97,7 +97,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -156,7 +156,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -208,7 +208,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -265,7 +265,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
}
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -332,7 +332,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -385,7 +385,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
}
url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -441,7 +441,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
params.placeholders = params.placeholders || {};
assert.response(server,
assert.response(self.server,
{
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),
method: 'POST',
@ -473,7 +473,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) :
url;
assert.response(server,
assert.response(self.server,
{
url: path,
method: 'POST',
@ -569,20 +569,22 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(server, request, expectedResponse, function(res, err) {
if (params.contentType) {
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var obj;
if (isPng) {
obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
}
else if (isMvt) {
} else if (isMvt) {
if (res.body) {
obj = new mapnik.VectorTile(z, x, y);
obj.setDataSync(new Buffer(res.body, 'binary'));
}
}
else {
} else {
obj = JSON.parse(res.body);
}
@ -618,7 +620,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -662,7 +664,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -723,7 +725,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
}
};
assert.response(server, request, expectedResponse, function(res, err) {
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
next(null, res, JSON.parse(res.body));
});
@ -741,6 +743,10 @@ TestClient.prototype.drain = function(callback) {
};
module.exports.getStaticMap = function getStaticMap(templateName, params, callback) {
var self = this;
self.server = new CartodbWindshaft(serverOptions);
if (!callback) {
callback = params;
params = null;
@ -771,9 +777,22 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
assert.response(self.server, requestOptions, expectedResponse, function (res, err) {
helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() {
return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary')));
});
});
};
TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) {
const userTimeoutLimitsKey = `limits:timeout:${user}`;
const params = [
userTimeoutLimitsKey,
'render', userTimeoutLimit,
'render_public', userTimeoutLimit
];
this.keysToDelete[userTimeoutLimitsKey] = 5;
helper.configureMetadata('hmset', params, callback);
}

19
test/support/test_helper.js
View File

@ -166,12 +166,29 @@ function rmdirRecursiveSync(dirname) {
}
}
function configureMetadata(action, params, callback) {
redisClient.SELECT(5, function (err) {
if (err) {
return callback(err);
}
redisClient[action](params, function (err) {
if (err) {
return callback(err);
}
return callback();
});
});
}
module.exports = {
deleteRedisKeys: deleteRedisKeys,
lzma_compress_to_base64: lzma_compress_to_base64,
checkNoCache: checkNoCache,
checkSurrogateKey: checkSurrogateKey,
checkCache: checkCache,
rmdirRecursiveSync: rmdirRecursiveSync
rmdirRecursiveSync: rmdirRecursiveSync,
configureMetadata
};