cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #717 from CartoDB/response-time-limit

Browse Source 
[limits] Response time limits
This commit is contained in:
Daniel 2017-08-10 17:35:47 +02:00 committed by GitHub
commit 43312922fc
15 changed files with 1772 additions and 424 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
lib/cartodb/api/user_limits_api.js
View File

@ -1,3 +1,5 @@
var step = require('step');
/**
*
* @param metadataBackend
@ -13,16 +15,65 @@ function UserLimitsApi(metadataBackend, options) {
module.exports = UserLimitsApi;
UserLimitsApi.prototype.getRenderLimits = function (username, callback) {
UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) {
var self = this;
this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) {
var limits = {
cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
render: self.options.limits.render || 0
};
self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) {
if (err) {
return callback(err);
}
return callback(null, {
cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
render: renderLimit || self.options.limits.render || 0
});
if (timeoutRenderLimit && timeoutRenderLimit.render) {
if (Number.isFinite(timeoutRenderLimit.render)) {
limits.render = timeoutRenderLimit.render;
}
}
return callback(null, limits);
});
};
UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
var self = this;
step(
function isAuthorized() {
var next = this;
if (!apiKey) {
return next(null, false);
}
self.metadataBackend.getUserMapKey(username, function (err, userApiKey) {
if (err) {
return next(err);
}
return next(null, userApiKey === apiKey);
});
},
function getUserTimeoutRenderLimits(err, authorized) {
var next = this;
if (err) {
return next(err);
}
self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) {
if (err) {
return next(err);
}
next(null, {
render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
});
});
},
callback
);
};

50
lib/cartodb/controllers/base.js
View File

@ -187,6 +187,9 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
BaseController.prototype.sendError = function(req, res, err, label) {
var allErrors = Array.isArray(err) ? err : [err];
allErrors = populateTimeoutErrors(allErrors);
label = label || 'UNKNOWN';
err = allErrors[0] || new Error(label);
allErrors[0] = err;
@ -221,6 +224,18 @@ function stripConnectionInfo(message) {
.replace(/is the server.*encountered/im, 'encountered');
}
var ERROR_INFO_TO_EXPOSE = {
message: true,
layer: true,
type: true,
analysis: true,
subtype: true
};
function shouldBeExposed (prop) {
return !!ERROR_INFO_TO_EXPOSE[prop];
}
function errorMessage(err) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var message = (_.isString(err) ? err : err.message) || 'Unknown error';
@ -239,7 +254,7 @@ function errorMessageWithContext(err) {
for (var prop in err) {
// type & message are properties from Error's prototype and will be skipped
if (err.hasOwnProperty(prop)) {
if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
error[prop] = err[prop];
}
}
@ -281,5 +296,38 @@ function statusFromErrorMessage(errMsg) {
statusCode = 404;
}
}
return statusCode;
}
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function populateTimeoutErrors (errors) {
return errors.map(function (error) {
if (isRenderTimeoutError(error)) {
error.subtype = 'render';
}
if (isDatasourceTimeoutError(error)) {
error.subtype = 'datasource';
}
if (isTimeoutError(error)) {
error.message = 'You are over platform\'s limits. Please contact us to know more details';
error.type = 'limit';
error.http_status = 429;
}
return error;
});
}

1
lib/cartodb/models/dataview/base.js
View File

@ -13,7 +13,6 @@ BaseDataview.prototype.getResult = function(psql, override, callback) {
}
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
}

2
lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js
View File

@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, this);
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

2
lib/cartodb/models/mapconfig/provider/map-store-provider.js
View File

@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, this);
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

2
lib/cartodb/models/mapconfig/provider/named-map-provider.js
View File

@ -114,7 +114,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
function prepareContextLimits(err, _mapConfig) {
assert.ifError(err);
mapConfig = _mapConfig;
self.userLimitsApi.getRenderLimits(self.owner, this);
self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this);
},
function cacheAndReturnMapConfig(err, renderLimits) {
self.err = err;

23
lib/cartodb/server.js
View File

@ -118,8 +118,27 @@ module.exports = function(serverOptions) {
var onTileErrorStrategy;
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
if (err && err.message === 'Render timed out' && format === 'png') {
return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRasterFormat (format) {
return format === 'png' || format === 'jpg';
}
if (isTimeoutError(err) && isRasterFormat(format)) {
return callback(null, timeoutErrorTile, {
'Content-Type': 'image/png',
}, {});
} else {
return callback(err, tile, headers, stats);
}

4
package.json
View File

@ -24,7 +24,7 @@
"camshaft": "0.55.6",
"cartodb-psql": "0.8.0",
"cartodb-query-tables": "0.2.0",
"cartodb-redis": "0.13.2",
"cartodb-redis": "0.14.0",
"debug": "~2.2.0",
"dot": "~1.0.2",
"express": "~4.13.3",
@ -40,7 +40,7 @@
"step-profiler": "~0.3.0",
"turbo-carto": "0.19.2",
"underscore": "~1.6.0",
"windshaft": "3.2.2",
"windshaft": "3.3.0",
"yargs": "~5.0.0"
},
"devDependencies": {

311
test/acceptance/limits.js
View File

@ -1,311 +0,0 @@
var testHelper = require('../support/test_helper');
var assert = require('../support/assert');
var _ = require('underscore');
var redis = require('redis');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var LayergroupToken = require('../support/layergroup-token');
describe('render limits', function() {
var layergroupUrl = '/api/v1/map';
var redisClient = redis.createClient(global.environment.redis.port);
var server;
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var user = 'localhost';
var pointSleepSql = "SELECT pg_sleep(0.5)," +
" 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id";
var pointCartoCss = '#layer { marker-fill:red; }';
var polygonSleepSql = "SELECT pg_sleep(0.5)," +
" ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id";
var polygonCartoCss = '#layer { polygon-fill:red; }';
function singleLayergroupConfig(sql, cartocss) {
return {
version: '1.0.0',
layers: [
{
type: 'mapnik',
options: {
sql: sql,
cartocss: cartocss,
cartocss_version: '2.0.1'
}
}
]
};
}
function createRequest(layergroup, userHost) {
return {
url: layergroupUrl,
method: 'POST',
headers: {
host: userHost,
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
};
}
function withRenderLimit(user, renderLimit, callback) {
redisClient.SELECT(5, function(err) {
if (err) {
return callback(err);
}
var userLimitsKey = 'limits:tiler:' + user;
redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) {
if (err) {
return callback(err);
}
keysToDelete[userLimitsKey] = 5;
return callback();
});
});
}
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategyEnabled;
before(function() {
onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
});
after(function() {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled;
});
it("layergroup creation fails if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed.errors, [ 'Render timed out' ]);
done();
}
);
});
});
it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done();
}
);
});
});
it("layergroup creation works if test tile is fast but tile request fails if they are slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed.errors, ['Render timed out']);
done();
}
);
}
);
});
});
it("tile request does not fail if user limit is high enough", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
done(err);
}
);
}
);
});
});
});
describe('with onTileErrorStrategy', function() {
it("layergroup creation works even if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done();
}
);
});
});
it("layergroup creation and tile requests works even if they are slow but returns fallback", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
if (err) {
done(err);
}
var referenceImagePath = './test/fixtures/render-timeout-fallback.png';
assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25,
function(imgErr/*, similarity*/) {
done(imgErr);
}
);
}
);
}
);
});
});
});
});

20
test/acceptance/mvt.js
View File

@ -24,14 +24,24 @@ describe('mvt', function () {
desc: 'should get empty mvt with code 204 (no content)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
status: 204,
response: {
status: 204,
headers: {
'Content-Type': undefined
}
},
mapConfig: createMapConfig(TestClient.SQL.EMPTY)
},
{
desc: 'should get mvt tile with code 200 (ok)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
status: 200,
response: {
status: 200,
headers: {
'Content-Type': 'application/x-protobuf'
}
},
mapConfig: createMapConfig()
}
];
@ -40,12 +50,12 @@ describe('mvt', function () {
it(test.desc, done => {
const testClient = new TestClient(test.mapConfig, 1234);
const { z, x, y } = test.coords;
const { format, status } = test;
const { format, response } = test;
testClient.getTile(z, x, y, { format, status }, (err, res) => {
testClient.getTile(z, x, y, { format, response }, (err, res) => {
assert.ifError(err);
assert.equal(res.statusCode, test.status);
assert.equal(res.statusCode, test.response.status);
testClient.drain(done);
});
});

786
test/acceptance/user-database-timeout-limit.js Normal file
View File

@ -0,0 +1,786 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
const pointSleepSql = `
SELECT
pg_sleep(0.3),
'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator,
1 cartodb_id,
2 val
`;
const validationPointSleepSql = `
SELECT
pg_sleep(0.3),
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = pointSleepSql,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id',
attributes
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
attributes,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
const DATASOURCE_TIMEOUT_ERROR = {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
};
describe('user database timeout limit', function () {
describe('dataview', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but dataview request fails due to statement timeout', function (done) {
const params = {
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getDataview('count', params, (err, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
describe('raster', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching raster tiles', function () {
describe('with user\'s timeout of 200 ms', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.setUserDatabaseTimeoutLimit(200, (err) => {
if (err) {
return done(err);
}
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('"png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'png',
layers: [ 0 ]
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
it('"static png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png'
};
this.testClient.getStaticCenter(params, function (err, res, tile) {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.setUserDatabaseTimeoutLimit(200, (err) => {
if (err) {
return done(err);
}
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('"png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'png',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
it('"static png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getStaticCenter(params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
});
describe('vector', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching vector tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"mvt" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'mvt',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('interactivity', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching interactivity tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ interactivity: 'val' });
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"grid.json" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'grid.json',
layers: 'mapnik',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('torque', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
type: 'torque',
cartocss: TestClient.CARTOCSS.TORQUE
});
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'torque-layer0', index: 0, type: 'torque' }
}]
});
done();
});
});
});
describe('fetching torque tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
type: 'torque',
cartocss: TestClient.CARTOCSS.TORQUE
});
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"torque.json" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'torque.json',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
it('".png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'torque.png',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => {
assert.ifError(err);
assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('attributes:', function () {
describe('while validating in map instatiation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
attributes: {
id: 'cartodb_id',
columns: [ 'val' ]
}
});
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: {
id: 'layer0',
index: 0,
type: 'mapnik'
}
}]
});
done();
});
});
});
describe('fetching by feature id', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
attributes: {
id: 'cartodb_id',
columns: [ 'val' ]
}
});
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
featureId: 1,
layer: 0,
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getAttributes(params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
});

394
test/acceptance/user-render-timeout-limit.js Normal file
View File

@ -0,0 +1,394 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
const pointSleepSql = `
SELECT
pg_sleep(0.5),
'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator,
1 cartodb_id,
2 val
`;
// during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout
const validationPointSleepSql = `
SELECT
pg_sleep(0.5),
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = pointSleepSql,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id'
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
describe('user render timeout limit', function () {
describe('map instantiation => validation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details",
layer: {
id: "layer0",
index: 0,
type: "mapnik"
}
}]
});
done();
});
});
});
describe('raster', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but tile request fails due to render timeout', function (done) {
this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works and render tile fails', function (done) {
var params = {
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details"
}]
});
done();
});
});
});
});
describe('vector', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but vector tile request fails due to render timeout', function (done) {
const params = {
format: 'mvt',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
});
done();
});
});
});
describe('interativity', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) {
const params = {
layers: 'mapnik',
format: 'grid.json',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
});
done();
});
});
});
describe('static images', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but static image fails due to render timeout', function (done) {
const params = {
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png'
};
this.testClient.getStaticCenter(params, function (err, res, tile) {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works and render tile fails', function (done) {
const params = {
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getStaticCenter(params, function (err, res, timeoutError) {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details"
}]
});
done();
});
});
});
});
});

404
test/support/test-client.js
View File

@ -3,7 +3,8 @@
var qs = require('querystring');
var step = require('step');
var urlParser = require('url');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
var mapnik = require('windshaft').mapnik;
var LayergroupToken = require('./layergroup-token');
@ -14,13 +15,21 @@ var helper = require('./test_helper');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
serverOptions.analysis.batch.inlineExecution = true;
var server = new CartodbWindshaft(serverOptions);
const MAPNIK_SUPPORTED_FORMATS = {
'png': true,
'png32': true,
'grid.json': true,
'geojson': true,
'mvt': true
}
function TestClient(config, apiKey) {
this.mapConfig = isMapConfig(config) ? config : null;
this.template = isTemplate(config) ? config : null;
this.apiKey = apiKey;
this.keysToDelete = {};
this.server = new CartodbWindshaft(serverOptions);
}
module.exports = TestClient;
@ -72,6 +81,34 @@ module.exports.CARTOCSS = {
' line-width: 0.5;',
' line-opacity: 1;',
'}'
].join('\n'),
TORQUE: [
'Map {',
' -torque-frame-count: 256;',
' -torque-animation-duration: 30;',
' -torque-time-attribute: "cartodb_id";',
' -torque-aggregation-function: "count(1)";',
' -torque-resolution: 4;',
' -torque-data-aggregation: linear;',
'}',
'#layer {',
' marker-width: 7;',
' marker-fill: #FFB927;',
' marker-fill-opacity: 0.9;',
' marker-line-width: 1;',
' marker-line-color: #FFF;',
' marker-line-opacity: 1;',
' comp-op: lighter;',
'}',
'#layer[frame-offset=1] {',
' marker-width: 9;',
' marker-fill-opacity: 0.45;',
'}',
'#layer[frame-offset=2] {',
' marker-width: 11;',
' marker-fill-opacity: 0.225;',
'}'
].join('\n')
};
@ -97,7 +134,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -156,7 +193,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -208,7 +245,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -265,7 +302,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
}
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -332,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -385,7 +422,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
}
url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -404,12 +441,16 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
);
},
function finish(err, dataview) {
if (err) {
return callback(err);
}
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, dataview);
return callback(null, dataview);
}
);
};
@ -446,7 +487,7 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params,
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -485,7 +526,7 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params,
url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@ -529,6 +570,11 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
var layergroupId;
if (params.layergroupid) {
layergroupId = params.layergroupid
}
step(
function createTemplate () {
var next = this;
@ -543,7 +589,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
params.placeholders = params.placeholders || {};
assert.response(server,
assert.response(self.server,
{
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),
method: 'POST',
@ -570,12 +616,16 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
function createLayergroup(err, templateId) {
var next = this;
if (layergroupId) {
return next(null, layergroupId);
}
var data = templateId ? params.placeholders : self.mapConfig
var path = templateId ?
urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) :
url;
assert.response(server,
assert.response(self.server,
{
url: path,
method: 'POST',
@ -616,6 +666,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
var format = params.format || 'png';
if (layers === undefined && !MAPNIK_SUPPORTED_FORMATS[format]) {
throw new Error(`Missing layer filter while fetching ${format} tile, review params argument`);
}
url += [z,x,y].join('/');
url += '.' + format;
@ -631,32 +685,30 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
};
var expectedResponse = {
status: params.status || 200,
var expectedResponse = Object.assign({}, {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
'Content-Type': 'image/png'
}
};
}, params.response);
var isPng = format.match(/png$/);
if (isPng) {
request.encoding = 'binary';
expectedResponse.headers['Content-Type'] = 'image/png';
}
var isMvt = format.match(/mvt$/);
if (isMvt) {
request.encoding = 'binary';
if (expectedResponse.status === 200) {
expectedResponse.headers['Content-Type'] = 'application/x-protobuf';
} else if (expectedResponse.status === 204) {
expectedResponse.headers['Content-Type'] = undefined;
}
}
var isGeojson = format.match(/geojson$/);
if (isGeojson) {
@ -671,29 +723,38 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(server, request, expectedResponse, function(res, err) {
if (params.contentType) {
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var obj;
if (isPng) {
obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
}
else if (isMvt) {
if (res.body) {
obj = new mapnik.VectorTile(z, x, y);
obj.setDataSync(new Buffer(res.body, 'binary'));
}
}
else {
obj = JSON.parse(res.body);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/x-protobuf':
body = new mapnik.VectorTile(z, x, y);
body.setDataSync(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, obj);
next(null, res, body);
});
},
function finish(err, res, image) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, image);
}
);
@ -718,7 +779,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -743,7 +804,111 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
return callback(err);
}
return callback(null, JSON.parse(res.body));
return callback(null, parsedBody);
}
);
};
TestClient.prototype.getStaticCenter = function (params, callback) {
var self = this;
let { layergroupid, z, lat, lng, width, height, format } = params
var url = `/api/v1/map/`;
if (this.apiKey) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
step(
function createLayergroup() {
var next = this;
if (layergroupid) {
return next(null, layergroupid);
}
var data = self.mapConfig
var path = url;
assert.response(self.server,
{
url: path,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(data)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
return next(null, JSON.parse(res.body).layergroupid);
}
);
},
function getStaticResult(err, _layergroupid) {
assert.ifError(err);
var next = this;
layergroupid = _layergroupid;
url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}`
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
}
var request = {
url: url,
encoding: 'binary',
method: 'GET',
headers: {
host: 'localhost'
}
};
var expectedResponse = Object.assign({}, {
status: 200,
headers: {
'Content-Type': 'image/png'
}
}, params.response);
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, body);
});
},
function finish(err, res, image) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, image);
}
);
};
@ -762,7 +927,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@ -823,7 +988,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
}
};
assert.response(server, request, expectedResponse, function(res, err) {
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
next(null, res, JSON.parse(res.body));
});
@ -836,11 +1001,119 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
);
};
TestClient.prototype.getAttributes = function(params, callback) {
var self = this;
if (!Number.isFinite(params.featureId)) {
throw new Error('featureId param must be a number')
}
if (!Number.isFinite(params.layer)) {
throw new Error('layer param must be a number')
}
var url = '/api/v1/map';
if (this.apiKey) {
url += '?' + qs.stringify({ api_key: this.apiKey });
}
var layergroupid;
if (params.layergroupid) {
layergroupid = params.layergroupid
}
step(
function createLayergroup() {
var next = this;
if (layergroupid) {
return next(null, layergroupid);
}
assert.response(self.server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
return next(null, parsedBody.layergroupid);
}
);
},
function getAttributes(err, _layergroupid) {
assert.ifError(err);
var next = this;
layergroupid = _layergroupid;
url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`;
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
}
var request = {
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
};
var expectedResponse = params.response || {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
assert.response(self.server, request, expectedResponse, function (res, err) {
assert.ifError(err);
var attributes = JSON.parse(res.body);
next(null, res, attributes);
});
},
function finish(err, res, attributes) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, attributes);
}
);
};
TestClient.prototype.drain = function(callback) {
helper.deleteRedisKeys(this.keysToDelete, callback);
};
module.exports.getStaticMap = function getStaticMap(templateName, params, callback) {
var self = this;
self.server = new CartodbWindshaft(serverOptions);
if (!callback) {
callback = params;
params = null;
@ -871,9 +1144,56 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
assert.response(self.server, requestOptions, expectedResponse, function (res, err) {
helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() {
return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary')));
});
});
};
TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) {
const userTimeoutLimitsKey = `limits:timeout:${user}`;
const params = [
userTimeoutLimitsKey,
'render', userTimeoutLimit,
'render_public', userTimeoutLimit
];
this.keysToDelete[userTimeoutLimitsKey] = 5;
helper.configureMetadata('hmset', params, callback);
};
TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) {
const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db';
const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 })
const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 })
const publicuser = global.environment.postgres.user;
// we need to guarantee all new connections have the new settings
helper.cleanPGPoolConnections()
const psql = new PSQL({
user: 'postgres',
dbname: dbname,
host: global.environment.postgres.host,
port: global.environment.postgres.port
});
step(
function configureTimeouts () {
const timeoutSQLs = [
`ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`
];
const group = this.group();
timeoutSQLs.forEach(sql => psql.query(sql, group()));
},
callback
);
};

26
test/support/test_helper.js
View File

@ -14,6 +14,7 @@ var lzmaWorker = new LZMA();
var redis = require('redis');
var nock = require('nock');
var log4js = require('log4js');
var pg = require('pg');
// set environment specific variables
global.environment = require(__dirname + '/../../config/environments/test');
@ -127,6 +128,11 @@ afterEach(function(done) {
});
});
function cleanPGPoolConnections () {
// TODO: this method will be replaced by psql.end
pg.end();
}
function deleteRedisKeys(keysToDelete, callback) {
if (Object.keys(keysToDelete).length === 0) {
@ -166,12 +172,30 @@ function rmdirRecursiveSync(dirname) {
}
}
function configureMetadata(action, params, callback) {
redisClient.SELECT(5, function (err) {
if (err) {
return callback(err);
}
redisClient[action](params, function (err) {
if (err) {
return callback(err);
}
return callback();
});
});
}
module.exports = {
deleteRedisKeys: deleteRedisKeys,
lzma_compress_to_base64: lzma_compress_to_base64,
checkNoCache: checkNoCache,
checkSurrogateKey: checkSurrogateKey,
checkCache: checkCache,
rmdirRecursiveSync: rmdirRecursiveSync
rmdirRecursiveSync: rmdirRecursiveSync,
configureMetadata,
cleanPGPoolConnections
};

108
yarn.lock
View File

@ -223,9 +223,9 @@ carto@0.16.3:
semver "^5.1.0"
yargs "^4.2.0"
carto@CartoDB/carto#0.15.1-cdb1:
"carto@github:cartodb/carto#0.15.1-cdb1":
version "0.15.1-cdb1"
resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
dependencies:
mapnik-reference "~6.0.2"
optimist "~0.6.0"
@ -254,13 +254,21 @@ cartodb-psql@0.8.0:
step "~0.0.6"
underscore "~1.6.0"
cartodb-psql@0.9.0:
version "0.9.0"
resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.9.0.tgz#887ba006fb499ab789c15d10f34f6afbe1f2f68d"
dependencies:
debug "~2.2.0"
pg cartodb/node-postgres#6.1.2-cdb1
underscore "~1.6.0"
cartodb-query-tables@0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44"
cartodb-redis@0.13.2:
version "0.13.2"
resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.13.2.tgz#de5214fa5c3ab336c4da978133efa8f908b3691c"
cartodb-redis@0.14.0:
version "0.14.0"
resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.14.0.tgz#6f82fdb3e5b7c8005dbaccd6172c1706c4378df2"
dependencies:
dot "~1.0.2"
redis-mpool "~0.4.1"
@ -1312,7 +1320,7 @@ mv@~2:
ncp "~2.0.0"
rimraf "~2.4.0"
nan@^2.0.8, nan@^2.3.4, nan@^2.4.0:
nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.6.2:
version "2.6.2"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45"
@ -1343,7 +1351,7 @@ nock@~2.11.0:
mkdirp "^0.5.0"
propagate "0.3.x"
node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.31:
node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36:
version "0.6.36"
resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786"
dependencies:
@ -1520,7 +1528,7 @@ pg-types@1.*:
postgres-date "~1.0.0"
postgres-interval "~1.0.0"
pg@cartodb/node-postgres#6.1.2-cdb1:
pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1":
version "6.1.2"
resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689"
dependencies:
@ -1741,32 +1749,7 @@ repeat-string@^1.5.2:
version "1.6.1"
resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0:
version "2.79.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
dependencies:
aws-sign2 "~0.6.0"
aws4 "^1.2.1"
caseless "~0.11.0"
combined-stream "~1.0.5"
extend "~3.0.0"
forever-agent "~0.6.1"
form-data "~2.1.1"
har-validator "~2.0.6"
hawk "~3.1.3"
http-signature "~1.1.0"
is-typedarray "~1.0.0"
isstream "~0.1.2"
json-stringify-safe "~5.0.1"
mime-types "~2.1.7"
oauth-sign "~0.8.1"
qs "~6.3.0"
stringstream "~0.0.4"
tough-cookie "~2.3.0"
tunnel-agent "~0.4.1"
uuid "^3.0.0"
request@^2.81.0:
request@2.x, request@^2.81.0:
version "2.81.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0"
dependencies:
@ -1793,6 +1776,31 @@ request@^2.81.0:
tunnel-agent "^0.6.0"
uuid "^3.0.0"
request@^2.55.0, request@^2.69.0, request@~2.79.0:
version "2.79.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
dependencies:
aws-sign2 "~0.6.0"
aws4 "^1.2.1"
caseless "~0.11.0"
combined-stream "~1.0.5"
extend "~3.0.0"
forever-agent "~0.6.1"
form-data "~2.1.1"
har-validator "~2.0.6"
hawk "~3.1.3"
http-signature "~1.1.0"
is-typedarray "~1.0.0"
isstream "~0.1.2"
json-stringify-safe "~5.0.1"
mime-types "~2.1.7"
oauth-sign "~0.8.1"
qs "~6.3.0"
stringstream "~0.0.4"
tough-cookie "~2.3.0"
tunnel-agent "~0.4.1"
uuid "^3.0.0"
require-directory@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
@ -1964,11 +1972,11 @@ sprintf-js@~1.0.2:
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
"sqlite3@2.x || 3.x":
version "3.1.8"
resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.8.tgz#4cbcf965d8b901d1b1015cbc7fc415aae157dfaa"
version "3.1.9"
resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.9.tgz#b2e7fbaa348380318d3834323918c3c351b8bf18"
dependencies:
nan "~2.4.0"
node-pre-gyp "~0.6.31"
nan "~2.6.2"
node-pre-gyp "~0.6.36"
srs@1.x:
version "1.2.0"
@ -2103,17 +2111,17 @@ through@2:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb3:
version "2.3.1-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23"
"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4":
version "2.3.1-cdb4"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6"
dependencies:
mapnik "~3.5.0"
mapnik-pool "~0.1.3"
sphericalmercator "1.0.x"
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2:
version "0.6.18-cdb2"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858"
"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3":
version "0.6.18-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01"
dependencies:
generic-pool "~2.4.0"
mapnik "3.5.14"
@ -2274,14 +2282,14 @@ window-size@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@3.2.2:
version "3.2.2"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.2.tgz#7afb9d8fd8bba1bf02d39c06e8bbe5a451aad953"
windshaft@3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.0.tgz#bf8ef35f9a2bb126934531b0020f5e8f96b37dfa"
dependencies:
abaculus cartodb/abaculus#2.0.3-cdb1
canvas cartodb/node-canvas#1.6.2-cdb2
carto cartodb/carto#0.15.1-cdb3
cartodb-psql "0.8.0"
cartodb-psql "0.9.0"
debug "~2.2.0"
dot "~1.0.2"
grainstore "~1.6.0"
@ -2293,8 +2301,8 @@ windshaft@3.2.2:
sphericalmercator "1.0.4"
step "~0.0.6"
tilelive "5.12.2"
tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb3
tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb2
tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb4
tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb3
torque.js "~2.11.0"
underscore "~1.6.0"