cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #283 from CartoDB/render-limits

Browse Source 
Render limits
This commit is contained in:
Raul Ochoa 2015-04-08 11:32:14 +02:00
commit 0062ec99f1
15 changed files with 2949 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.jshintrc
View File

@ -87,6 +87,8 @@
"describe": true,
"before": true,
"after": true,
"beforeEach": true,
"afterEach": true,
"it": true,
"suite": true,
"suiteSetup": true,

2
Makefile
View File

@ -29,7 +29,7 @@ test: config/environments/test.js
jshint:
@echo "***jshint***"
@./node_modules/.bin/jshint lib/
@./node_modules/.bin/jshint lib/ app.js
test-all: jshint test

2
NEWS.md
View File

@ -24,6 +24,8 @@ Announcements:
- scale_factor
* Affected tables for x-cache-channel will use direct connection to postgresql
* Removes some metrics: authorized times ones
* Mapnik renderer configuration not part of the `renderer` root configuration
- All configuration must be moved into `renderer.mapnik`, see `config/environments/*.js.example` for reference
- Removes rollbar as optional logger

45
app.js
View File

@ -7,17 +7,21 @@
* environments: [development, production]
*/
var path = require('path'),
fs = require('fs'),
RedisPool = require('redis-mpool')
;
var path = require('path');
var fs = require('fs');
var RedisPool = require('redis-mpool');
var _ = require('underscore');
var ENV;
if ( process.argv[2] ) {
ENV = process.argv[2];
} else if ( process.env.NODE_ENV ) {
ENV = process.env.NODE_ENV;
} else {
ENV = 'development';
}
if ( process.argv[2] ) ENV = process.argv[2];
else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
else ENV = 'development';
process.env['NODE_ENV'] = ENV;
process.env.NODE_ENV = ENV;
// sanity check
if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
@ -26,14 +30,12 @@ if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
process.exit(1);
}
var _ = require('underscore');
// set environment specific variables
global.environment = require(__dirname + '/config/environments/' + ENV);
global.environment.api_hostname = require('os').hostname().split('.')[0];
global.log4js = require('log4js');
log4js_config = {
var log4js_config = {
appenders: [],
replaceConsole:true
};
@ -60,18 +62,18 @@ if ( global.environment.log_filename ) {
);
}
log4js.configure(log4js_config, { cwd: __dirname });
global.logger = log4js.getLogger();
global.log4js.configure(log4js_config, { cwd: __dirname });
global.logger = global.log4js.getLogger();
var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
redisPool = new RedisPool(redisOpts);
// Include cartodb_windshaft only _after_ the "global" variable is set
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
var cartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
serverOptions = require('./lib/cartodb/server_options')(redisPool);
ws = CartodbWindshaft(serverOptions);
var ws = cartodbWindshaft(serverOptions);
if (global.statsClient) {
redisPool.on('status', function(status) {
@ -93,19 +95,20 @@ ws.listen(global.environment.port, global.environment.host);
var version = require("./package").version;
ws.on('listening', function() {
console.log("Windshaft tileserver " + version + " started on "
+ global.environment.host + ':' + global.environment.port
+ " (" + ENV + ")");
console.log(
"Windshaft tileserver %s started on %s:%s (%s)",
version, global.environment.host, global.environment.port, ENV
);
});
process.on('SIGHUP', function() {
global.log4js.clearAndShutdownAppenders(function() {
global.log4js.configure(log4js_config);
global.logger = log4js.getLogger();
global.logger = global.log4js.getLogger();
console.log('Log files reloaded');
});
});
process.on('uncaughtException', function(err) {
logger.error('Uncaught exception: ' + err.stack);
global.logger.error('Uncaught exception: ' + err.stack);
});

BIN
assets/render-timeout-fallback.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.9 KiB

46
config/environments/development.js.example
View File

@ -84,9 +84,49 @@ var config = {
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
// to be rendered at once. If all of them will be requested
// we'd have saved time. If only one will be used, we'd have
// wasted time.
metatile: 2,
// Buffer size is the tickness in pixel of a buffer
// around the rendered (meta?)tile.
//
// This is important for labels and other marker that overlap tile boundaries.
// Setting to 128 ensures no render artifacts.
// 64 may have artifacts but is faster.
// Less important if we can turn metatiling on.
bufferSize: 64,
// SQL queries will be wrapped with ST_SnapToGrid
// Snapping all points of the geometry to a regular grid
snapToGrid: false,
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
// - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
render: 0,
// As the render request will finish even if timed out, whether it should be placed in the internal
// cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
// the same tile will result in an immediate response, however that will use a lot of more application
// memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
// internal cache.
cacheOnTimeout: true
}
},
http: {
timeout: 2000, // the timeout in ms for a http tile request
proxy: undefined, // the url for a proxy server
@ -156,6 +196,8 @@ var config = {
// Use this as a feature flags enabling/disabling mechanism
,enabledFeatures: {
// whether it should intercept tile render errors an act based on them, enabled by default.
onTileErrorStrategy: true,
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true
}

46
config/environments/production.js.example
View File

@ -78,9 +78,49 @@ var config = {
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
// to be rendered at once. If all of them will be requested
// we'd have saved time. If only one will be used, we'd have
// wasted time.
metatile: 2,
// Buffer size is the tickness in pixel of a buffer
// around the rendered (meta?)tile.
//
// This is important for labels and other marker that overlap tile boundaries.
// Setting to 128 ensures no render artifacts.
// 64 may have artifacts but is faster.
// Less important if we can turn metatiling on.
bufferSize: 64,
// SQL queries will be wrapped with ST_SnapToGrid
// Snapping all points of the geometry to a regular grid
snapToGrid: false,
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
// - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
render: 0,
// As the render request will finish even if timed out, whether it should be placed in the internal
// cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
// the same tile will result in an immediate response, however that will use a lot of more application
// memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
// internal cache.
cacheOnTimeout: true
}
},
http: {
timeout: 2000, // the timeout in ms for a http tile request
proxy: undefined, // the url for a proxy server
@ -156,6 +196,8 @@ var config = {
// Use this as a feature flags enabling/disabling mechanism
,enabledFeatures: {
// whether it should intercept tile render errors an act based on them, enabled by default.
onTileErrorStrategy: true,
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true
}

46
config/environments/staging.js.example
View File

@ -78,9 +78,49 @@ var config = {
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
// to be rendered at once. If all of them will be requested
// we'd have saved time. If only one will be used, we'd have
// wasted time.
metatile: 2,
// Buffer size is the tickness in pixel of a buffer
// around the rendered (meta?)tile.
//
// This is important for labels and other marker that overlap tile boundaries.
// Setting to 128 ensures no render artifacts.
// 64 may have artifacts but is faster.
// Less important if we can turn metatiling on.
bufferSize: 64,
// SQL queries will be wrapped with ST_SnapToGrid
// Snapping all points of the geometry to a regular grid
snapToGrid: false,
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
// - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
render: 0,
// As the render request will finish even if timed out, whether it should be placed in the internal
// cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
// the same tile will result in an immediate response, however that will use a lot of more application
// memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
// internal cache.
cacheOnTimeout: true
}
},
http: {
timeout: 2000, // the timeout in ms for a http tile request
proxy: undefined, // the url for a proxy server
@ -156,6 +196,8 @@ var config = {
// Use this as a feature flags enabling/disabling mechanism
,enabledFeatures: {
// whether it should intercept tile render errors an act based on them, enabled by default.
onTileErrorStrategy: true,
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true
}

46
config/environments/test.js.example
View File

@ -78,9 +78,49 @@ var config = {
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
// to be rendered at once. If all of them will be requested
// we'd have saved time. If only one will be used, we'd have
// wasted time.
metatile: 2,
// Buffer size is the tickness in pixel of a buffer
// around the rendered (meta?)tile.
//
// This is important for labels and other marker that overlap tile boundaries.
// Setting to 128 ensures no render artifacts.
// 64 may have artifacts but is faster.
// Less important if we can turn metatiling on.
bufferSize: 64,
// SQL queries will be wrapped with ST_SnapToGrid
// Snapping all points of the geometry to a regular grid
snapToGrid: false,
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
// - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
render: 0,
// As the render request will finish even if timed out, whether it should be placed in the internal
// cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
// the same tile will result in an immediate response, however that will use a lot of more application
// memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
// internal cache.
cacheOnTimeout: true
}
},
http: {
timeout: 2000, // the timeout in ms for a http tile request
proxy: undefined, // the url for a proxy server
@ -152,6 +192,8 @@ var config = {
// Use this as a feature flags enabling/disabling mechanism
,enabledFeatures: {
// whether it should intercept tile render errors an act based on them, enabled by default.
onTileErrorStrategy: true,
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true
}

1
lib/cartodb/controllers/template_maps.js
View File

@ -409,6 +409,7 @@ TemplateMapsController.prototype.instantiateTemplate = function(req, res, templa
if ( err ) throw err;
layergroup = instance;
fakereq = {query: {}, params: {}, headers: _.clone(req.headers),
context: _.clone(req.context),
method: req.method,
res: res,
profiler: req.profiler

68
lib/cartodb/server_options.js
View File

@ -1,12 +1,17 @@
var _ = require('underscore');
var step = require('step');
var LZMA = require('lzma').LZMA;
var assert = require('assert');
var RedisPool = require('redis-mpool');
var QueryTablesApi = require('./api/query_tables_api');
var PgConnection = require('./backends/pg_connection');
var LZMA = require('lzma').LZMA;
var TemplateMaps = require('./template_maps.js');
var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter');
var CdbRequest = require('./models/cdb_request');
var assert = require('assert');
var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
// Whitelist query parameters and attach format
var REQUEST_QUERY_PARAMS_WHITELIST = [
@ -18,8 +23,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
];
module.exports = function(redisPool) {
redisPool = redisPool ||
require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:server_options'}));
redisPool = redisPool || new RedisPool(_.extend(global.environment.redis, {name: 'windshaft:server_options'}));
var cartoData = require('cartodb-redis')({ pool: redisPool }),
lzmaWorker = new LZMA(),
@ -29,9 +33,16 @@ module.exports = function(redisPool) {
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
metatile: 4,
statsInterval: 60000,
mapnik: {
poolSize: 8,
metatile: 2,
bufferSize: 64,
statsInterval: 60000
snapToGrid: false,
clipByBox2d: false,
limits: {}
},
http: {}
});
var me = {
@ -59,17 +70,13 @@ module.exports = function(redisPool) {
mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
default_layergroup_ttl: global.environment.mapConfigTTL || 7200
},
mapnik: {
poolSize: rendererConfig.poolSize,
metatile: rendererConfig.metatile,
bufferSize: rendererConfig.bufferSize
},
statsd: global.environment.statsd,
renderCache: {
ttl: rendererConfig.cache_ttl,
statsInterval: rendererConfig.statsInterval
},
renderer: {
mapnik: rendererConfig.mapnik,
http: rendererConfig.http
},
redis: global.environment.redis,
@ -240,6 +247,45 @@ module.exports = function(redisPool) {
});
};
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
me.renderer.onTileErrorStrategy = function(err, tile, headers, stats, format, callback) {
if (err && err.message === 'Render timed out' && format === 'png') {
return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
} else {
return callback(err, tile, headers, stats);
}
};
}
me.renderCache.beforeRendererCreate = function(req, callback) {
var user = cdbRequest.userByReq(req);
var rendererOptions = {};
step(
function getLimits(err) {
assert.ifError(err);
cartoData.getTilerRenderLimit(user, this);
},
function handleTilerLimits(err, renderLimit) {
assert.ifError(err);
rendererOptions.limits = {
cacheOnTimeout: rendererConfig.mapnik.limits.cacheOnTimeout || false,
render: renderLimit || rendererConfig.mapnik.limits.render || 0
};
return null;
},
function finish(err) {
if (err) {
return callback(err);
}
return callback(null, rendererOptions);
}
);
};
me.beforeLayergroupCreate = function(req, requestMapConfig, callback) {
mapConfigNamedLayersAdapter.getLayers(cdbRequest.userByReq(req), requestMapConfig.layers, pgConnection,
function(err, layers, datasource) {

2375
npm-shrinkwrap.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

4
package.json
View File

@ -24,11 +24,11 @@
"dependencies": {
"underscore" : "~1.6.0",
"dot": "~1.0.2",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/master",
"windshaft": "0.41.0",
"step": "~0.0.5",
"queue-async": "~1.0.7",
"request": "~2.9.203",
"cartodb-redis": "~0.11.0",
"cartodb-redis": "~0.12.1",
"cartodb-psql": "~0.4.0",
"redis-mpool": "~0.3.0",
"lzma": "~1.3.7",

307
test/acceptance/limits.js Normal file
View File

@ -0,0 +1,307 @@
require('../support/test_helper');
var assert = require('../support/assert');
var _ = require('underscore');
var redis = require('redis');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var serverOptions = require('../../lib/cartodb/server_options');
describe('render limits', function() {
var layergroupUrl = '/api/v1/map';
var redisClient = redis.createClient(global.environment.redis.port);
after(function(done) {
redisClient.keys("map_style|*", function(err, matches) {
redisClient.del(matches, function() {
done();
});
});
});
var server;
beforeEach(function() {
server = new CartodbWindshaft(serverOptions());
server.setMaxListeners(0);
});
var keysToDelete = [];
afterEach(function(done) {
redisClient.DEL(keysToDelete, function() {
keysToDelete = [];
done();
});
});
var user = 'localhost';
var pointSleepSql = "SELECT pg_sleep(0.5)," +
" 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id";
var pointCartoCss = '#layer { marker-fill:red; }';
var polygonSleepSql = "SELECT pg_sleep(0.5)," +
" ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id";
var polygonCartoCss = '#layer { polygon-fill:red; }';
function singleLayergroupConfig(sql, cartocss) {
return {
version: '1.0.0',
layers: [
{
type: 'mapnik',
options: {
sql: sql,
cartocss: cartocss,
cartocss_version: '2.0.1'
}
}
]
};
}
function createRequest(layergroup, userHost) {
return {
url: layergroupUrl,
method: 'POST',
headers: {
host: userHost,
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
};
}
function withRenderLimit(user, renderLimit, callback) {
redisClient.SELECT(5, function(err) {
if (err) {
return callback(err);
}
var userLimitsKey = 'limits:tiler:' + user;
redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) {
if (err) {
return callback(err);
}
keysToDelete.push(userLimitsKey);
return callback();
});
});
}
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategyEnabled;
before(function() {
onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
});
after(function() {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled;
});
it("layergroup creation fails if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed, { errors: [ 'Render timed out' ] });
done();
}
);
});
});
it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
done();
}
);
});
});
it("layergroup creation works if test tile is fast but tile request fails if they are slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed, { error: 'Render timed out' });
done();
}
);
}
);
});
});
it("tile request does not fail if user limit is high enough", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
done(err);
}
);
}
);
});
});
});
describe('with onTileErrorStrategy', function() {
it("layergroup creation works even if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
done();
}
);
});
});
it("layergroup creation and tile requests works even if they are slow but returns fallback", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
if (err) {
done(err);
}
assert.imageEqualsFile(res.body, './test/fixtures/render-timeout-fallback.png', 25,
function(imgErr/*, similarity*/) {
done(imgErr);
}
);
}
);
}
);
});
});
});
});

BIN
test/fixtures/render-timeout-fallback.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.9 KiB