Merge pull request #283 from CartoDB/render-limits

Render limits

.jshintrc

@@ -87,6 +87,8 @@
         "describe": true,
         "before": true,
         "after": true,
+        "beforeEach": true,
+        "afterEach": true,
         "it": true,
         "suite": true,
         "suiteSetup": true,

Makefile

@@ -29,7 +29,7 @@ test: config/environments/test.js
 
 jshint:
 	@echo "***jshint***"
-	@./node_modules/.bin/jshint lib/
+	@./node_modules/.bin/jshint lib/ app.js
 
 test-all: jshint test
 

NEWS.md

@@ -24,6 +24,8 @@ Announcements:
     - scale_factor
   * Affected tables for x-cache-channel will use direct connection to postgresql
   * Removes some metrics: authorized times ones
+  * Mapnik renderer configuration not part of the `renderer` root configuration
+    - All configuration must be moved into `renderer.mapnik`, see `config/environments/*.js.example` for reference
  - Removes rollbar as optional logger
 
 

app.js

@@ -7,17 +7,21 @@
  * environments: [development, production]
  */
 
-var path = require('path'),
+var path = require('path');
-    fs = require('fs'),
+var fs = require('fs');
-    RedisPool = require('redis-mpool')
+var RedisPool = require('redis-mpool');
-;
+var _ = require('underscore');
 
+var ENV;
+if ( process.argv[2] ) {
+    ENV = process.argv[2];
+} else if ( process.env.NODE_ENV ) {
+    ENV = process.env.NODE_ENV;
+} else {
+    ENV = 'development';
+}
 
-if ( process.argv[2] ) ENV = process.argv[2];
+process.env.NODE_ENV = ENV;
-else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
-else ENV = 'development';
-
-process.env['NODE_ENV'] = ENV;
 
 // sanity check
 if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
@@ -26,14 +30,12 @@ if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     process.exit(1);
 }
 
-var _ = require('underscore');
-
 // set environment specific variables
 global.environment  = require(__dirname + '/config/environments/' + ENV);
 global.environment.api_hostname = require('os').hostname().split('.')[0];
 
 global.log4js = require('log4js');
-log4js_config = {
+var log4js_config = {
   appenders: [],
   replaceConsole:true
 };
@@ -60,18 +62,18 @@ if ( global.environment.log_filename ) {
   );
 }
 
-log4js.configure(log4js_config, { cwd: __dirname });
+global.log4js.configure(log4js_config, { cwd: __dirname });
-global.logger = log4js.getLogger();
+global.logger = global.log4js.getLogger();
 
 var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
     redisPool = new RedisPool(redisOpts);
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
+var cartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
     serverOptions = require('./lib/cartodb/server_options')(redisPool);
 
-ws = CartodbWindshaft(serverOptions);
+var ws = cartodbWindshaft(serverOptions);
 
 if (global.statsClient) {
     redisPool.on('status', function(status) {
@@ -93,19 +95,20 @@ ws.listen(global.environment.port, global.environment.host);
 var version = require("./package").version;
 
 ws.on('listening', function() {
-  console.log("Windshaft tileserver " + version + " started on "
+  console.log(
-              + global.environment.host + ':' + global.environment.port
+      "Windshaft tileserver %s started on %s:%s (%s)",
-              + " (" + ENV + ")");
+      version, global.environment.host, global.environment.port, ENV
+  );
 });
 
 process.on('SIGHUP', function() {
     global.log4js.clearAndShutdownAppenders(function() {
         global.log4js.configure(log4js_config);
-        global.logger = log4js.getLogger();
+        global.logger = global.log4js.getLogger();
         console.log('Log files reloaded');
     });
 });
 
 process.on('uncaughtException', function(err) {
-  logger.error('Uncaught exception: ' + err.stack); 
+    global.logger.error('Uncaught exception: ' + err.stack);
 });

config/environments/development.js.example

@@ -84,9 +84,49 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik renderers
+          // Check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          }
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
@@ -156,6 +196,8 @@ var config = {
 
     // Use this as a feature flags enabling/disabling mechanism
     ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true
     }

config/environments/production.js.example

@@ -78,9 +78,49 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik renderers
+          // Check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          }
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
@@ -156,6 +196,8 @@ var config = {
 
     // Use this as a feature flags enabling/disabling mechanism
     ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true
     }

config/environments/staging.js.example

@@ -78,9 +78,49 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik renderers
+          // Check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          }
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
@@ -156,6 +196,8 @@ var config = {
 
     // Use this as a feature flags enabling/disabling mechanism
     ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true
     }

config/environments/test.js.example

@@ -78,9 +78,49 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik renderers
+          // Check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          }
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
@@ -152,6 +192,8 @@ var config = {
 
     // Use this as a feature flags enabling/disabling mechanism
     ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true
     }

lib/cartodb/controllers/template_maps.js

@@ -408,7 +408,8 @@ TemplateMapsController.prototype.instantiateTemplate = function(req, res, templa
             if ( req.profiler ) req.profiler.done('TemplateMaps_instance');
             if ( err ) throw err;
             layergroup = instance;
-            fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
+            fakereq = {query: {}, params: {}, headers: _.clone(req.headers),
+                context: _.clone(req.context),
                 method: req.method,
                 res: res,
                 profiler: req.profiler

lib/cartodb/server_options.js

@@ -1,12 +1,17 @@
 var _ = require('underscore');
 var step = require('step');
+var LZMA = require('lzma').LZMA;
+var assert = require('assert');
+var RedisPool = require('redis-mpool');
+
 var QueryTablesApi = require('./api/query_tables_api');
 var PgConnection = require('./backends/pg_connection');
-var LZMA = require('lzma').LZMA;
 var TemplateMaps = require('./template_maps.js');
 var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter');
 var CdbRequest = require('./models/cdb_request');
-var assert = require('assert');
+
+var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
+var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
 // Whitelist query parameters and attach format
 var REQUEST_QUERY_PARAMS_WHITELIST = [
@@ -18,8 +23,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
 ];
 
 module.exports = function(redisPool) {
-    redisPool = redisPool ||
+    redisPool = redisPool || new RedisPool(_.extend(global.environment.redis, {name: 'windshaft:server_options'}));
-        require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:server_options'}));
 
     var cartoData = require('cartodb-redis')({ pool: redisPool }),
         lzmaWorker = new LZMA(),
@@ -29,9 +33,16 @@ module.exports = function(redisPool) {
 
     var rendererConfig = _.defaults(global.environment.renderer || {}, {
         cache_ttl: 60000, // milliseconds
-        metatile: 4,
+        statsInterval: 60000,
-        bufferSize: 64,
+        mapnik: {
-        statsInterval: 60000
+            poolSize: 8,
+            metatile: 2,
+            bufferSize: 64,
+            snapToGrid: false,
+            clipByBox2d: false,
+            limits: {}
+        },
+        http: {}
     });
 
     var me = {
@@ -59,17 +70,13 @@ module.exports = function(redisPool) {
           mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
           default_layergroup_ttl: global.environment.mapConfigTTL || 7200
         },
-        mapnik: {
-          poolSize: rendererConfig.poolSize,
-          metatile: rendererConfig.metatile,
-          bufferSize: rendererConfig.bufferSize
-        },
         statsd: global.environment.statsd,
         renderCache: {
             ttl: rendererConfig.cache_ttl,
             statsInterval: rendererConfig.statsInterval
         },
         renderer: {
+            mapnik: rendererConfig.mapnik,
             http: rendererConfig.http
         },
         redis: global.environment.redis,
@@ -240,6 +247,45 @@ module.exports = function(redisPool) {
         });
     };
 
+
+    if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
+        me.renderer.onTileErrorStrategy = function(err, tile, headers, stats, format, callback) {
+            if (err && err.message === 'Render timed out' && format === 'png') {
+                return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
+            } else {
+                return callback(err, tile, headers, stats);
+            }
+        };
+    }
+
+    me.renderCache.beforeRendererCreate = function(req, callback) {
+        var user = cdbRequest.userByReq(req);
+
+        var rendererOptions = {};
+
+        step(
+            function getLimits(err) {
+                assert.ifError(err);
+                cartoData.getTilerRenderLimit(user, this);
+            },
+            function handleTilerLimits(err, renderLimit) {
+                assert.ifError(err);
+                rendererOptions.limits = {
+                    cacheOnTimeout: rendererConfig.mapnik.limits.cacheOnTimeout || false,
+                    render: renderLimit || rendererConfig.mapnik.limits.render || 0
+                };
+                return null;
+            },
+            function finish(err) {
+                if (err) {
+                    return callback(err);
+                }
+
+                return callback(null, rendererOptions);
+            }
+        );
+    };
+
     me.beforeLayergroupCreate = function(req, requestMapConfig, callback) {
         mapConfigNamedLayersAdapter.getLayers(cdbRequest.userByReq(req), requestMapConfig.layers, pgConnection,
             function(err, layers, datasource) {

package.json

@@ -24,11 +24,11 @@
     "dependencies": {
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/master",
+        "windshaft": "0.41.0",
         "step": "~0.0.5",
         "queue-async": "~1.0.7",
         "request": "~2.9.203",
-        "cartodb-redis": "~0.11.0",
+        "cartodb-redis": "~0.12.1",
         "cartodb-psql": "~0.4.0",
         "redis-mpool": "~0.3.0",
         "lzma": "~1.3.7",

test/acceptance/limits.js

@@ -0,0 +1,307 @@
+require('../support/test_helper');
+
+var assert = require('../support/assert');
+var _ = require('underscore');
+var redis = require('redis');
+
+var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
+var serverOptions = require('../../lib/cartodb/server_options');
+
+describe('render limits', function() {
+
+    var layergroupUrl = '/api/v1/map';
+
+    var redisClient = redis.createClient(global.environment.redis.port);
+    after(function(done) {
+        redisClient.keys("map_style|*", function(err, matches) {
+            redisClient.del(matches, function() {
+                done();
+            });
+        });
+    });
+
+    var server;
+    beforeEach(function() {
+        server = new CartodbWindshaft(serverOptions());
+        server.setMaxListeners(0);
+    });
+
+    var keysToDelete = [];
+    afterEach(function(done) {
+        redisClient.DEL(keysToDelete, function() {
+            keysToDelete = [];
+            done();
+        });
+    });
+
+    var user = 'localhost';
+
+    var pointSleepSql = "SELECT pg_sleep(0.5)," +
+        " 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id";
+    var pointCartoCss = '#layer { marker-fill:red; }';
+    var polygonSleepSql = "SELECT pg_sleep(0.5)," +
+        " ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id";
+    var polygonCartoCss = '#layer { polygon-fill:red; }';
+
+    function singleLayergroupConfig(sql, cartocss) {
+        return {
+            version: '1.0.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: sql,
+                        cartocss: cartocss,
+                        cartocss_version: '2.0.1'
+                    }
+                }
+            ]
+        };
+    }
+
+    function createRequest(layergroup, userHost) {
+        return {
+            url: layergroupUrl,
+            method: 'POST',
+            headers: {
+                host: userHost,
+                'Content-Type': 'application/json'
+            },
+            data: JSON.stringify(layergroup)
+        };
+    }
+
+    function withRenderLimit(user, renderLimit, callback) {
+        redisClient.SELECT(5, function(err) {
+            if (err) {
+                return callback(err);
+            }
+            var userLimitsKey = 'limits:tiler:' + user;
+            redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) {
+                if (err) {
+                    return callback(err);
+                }
+                keysToDelete.push(userLimitsKey);
+                return callback();
+            });
+        });
+
+    }
+
+    describe('with onTileErrorStrategy DISABLED', function() {
+        var onTileErrorStrategyEnabled;
+        before(function() {
+            onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy;
+            global.environment.enabledFeatures.onTileErrorStrategy = false;
+        });
+
+        after(function() {
+            global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled;
+        });
+
+        it("layergroup creation fails if test tile is slow", function(done) {
+            withRenderLimit(user, 50, function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
+                assert.response(server,
+                    createRequest(layergroup, user),
+                    {
+                        status: 400
+                    },
+                    function(res) {
+                        var parsed = JSON.parse(res.body);
+                        assert.deepEqual(parsed, { errors: [ 'Render timed out' ] });
+                        done();
+                    }
+                );
+            });
+        });
+
+        it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) {
+            withRenderLimit(user, 5000, function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
+                assert.response(server,
+                    createRequest(layergroup, user),
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        var parsed = JSON.parse(res.body);
+                        assert.ok(parsed.layergroupid);
+                        done();
+                    }
+                );
+            });
+        });
+
+
+        it("layergroup creation works if test tile is fast but tile request fails if they are slow",  function(done) {
+            withRenderLimit(user, 50, function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
+                assert.response(server,
+                    createRequest(layergroup, user),
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        assert.response(server,
+                            {
+                                url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
+                                    layergroupId: JSON.parse(res.body).layergroupid,
+                                    z: 0,
+                                    x: 0,
+                                    y: 0
+                                }),
+                                method: 'GET',
+                                headers: {
+                                    host: 'localhost'
+                                },
+                                encoding: 'binary'
+                            },
+                            {
+                                status: 400
+                            },
+                            function(res) {
+                                var parsed = JSON.parse(res.body);
+                                assert.deepEqual(parsed, { error: 'Render timed out' });
+                                done();
+                            }
+                        );
+
+                    }
+                );
+            });
+        });
+
+        it("tile request does not fail if user limit is high enough",  function(done) {
+            withRenderLimit(user, 5000, function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
+                assert.response(server,
+                    createRequest(layergroup, user),
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        assert.response(server,
+                            {
+                                url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
+                                    layergroupId: JSON.parse(res.body).layergroupid,
+                                    z: 0,
+                                    x: 0,
+                                    y: 0
+                                }),
+                                method: 'GET',
+                                headers: {
+                                    host: 'localhost'
+                                },
+                                encoding: 'binary'
+                            },
+                            {
+                                status: 200,
+                                headers: {
+                                    'Content-Type': 'image/png'
+                                }
+                            },
+                            function(res, err) {
+                                done(err);
+                            }
+                        );
+
+                    }
+                );
+            });
+        });
+
+    });
+
+    describe('with onTileErrorStrategy', function() {
+
+        it("layergroup creation works even if test tile is slow", function(done) {
+            withRenderLimit(user, 50, function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
+                assert.response(server,
+                    createRequest(layergroup, user),
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        var parsed = JSON.parse(res.body);
+                        assert.ok(parsed.layergroupid);
+                        done();
+                    }
+                );
+            });
+        });
+
+        it("layergroup creation and tile requests works even if they are slow but returns fallback",  function(done) {
+            withRenderLimit(user, 50, function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
+                assert.response(server,
+                    createRequest(layergroup, user),
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        assert.response(server,
+                            {
+                                url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
+                                    layergroupId: JSON.parse(res.body).layergroupid,
+                                    z: 0,
+                                    x: 0,
+                                    y: 0
+                                }),
+                                method: 'GET',
+                                headers: {
+                                    host: 'localhost'
+                                },
+                                encoding: 'binary'
+                            },
+                            {
+                                status: 200,
+                                headers: {
+                                    'Content-Type': 'image/png'
+                                }
+                            },
+                            function(res, err) {
+                                if (err) {
+                                    done(err);
+                                }
+                                assert.imageEqualsFile(res.body, './test/fixtures/render-timeout-fallback.png', 25,
+                                    function(imgErr/*, similarity*/) {
+                                        done(imgErr);
+                                    }
+                                );
+                            }
+                        );
+
+                    }
+                );
+            });
+        });
+
+    });
+
+});