Merge branch 'standalone-server' into standalone-server-mvt

CONTRIBUTING.md

@@ -0,0 +1,11 @@
+Contributing
+---
+
+The issue tracker is at [github.com/CartoDB/Windshaft-cartodb](https://github.com/CartoDB/Windshaft-cartodb).
+
+We love pull requests from everyone, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
+
+
+## Submitting Contributions
+
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014, Vizzuality
+Copyright (c) 2015, CartoDB
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without 

Makefile

@@ -22,6 +22,7 @@ test: config/environments/test.js
 	@echo "***tests***"
 	@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \
 		test/unit/cartodb/*.js \
+		test/unit/cartodb/ported/*.js \
 		test/unit/cartodb/cache/model/*.js \
 		test/integration/*.js \
 		test/acceptance/*.js \
@@ -33,6 +34,7 @@ test-unit: config/environments/test.js
 	@echo "***tests***"
 	@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \
 		test/unit/cartodb/*.js \
+		test/unit/cartodb/ported/*.js \
 		test/unit/cartodb/cache/model/*.js
 
 test-integration: config/environments/test.js

README.md

@@ -95,3 +95,9 @@ Examples
 --------
 
 [CartoDB's Map Gallery](http://cartodb.com/gallery/) showcases several examples of visualisations built on top of this.
+
+
+Contributing
+---
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).

lib/cartodb/controllers/layergroup.js

@@ -55,20 +55,18 @@ LayergroupController.prototype.attributes = function(req, res) {
             self.app.req2params(req, this);
         },
         function retrieveFeatureAttributes(err) {
-            req.profiler.done('req2params');
-
             assert.ifError(err);
 
-            self.attributesBackend.getFeatureAttributes(req.params, false, this);
+            var mapConfigProvider = new MapStoreMapConfigProvider(
+                self.mapStore, req.context.user, self.userLimitsApi, req.params
+            );
+            self.attributesBackend.getFeatureAttributes(mapConfigProvider, req.params, false, this);
         },
         function finish(err, tile, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+                res.sendError(err, 'GET ATTRIBUTES');
-                var errMsg = err.message ? ( '' + err.message ) : ( '' + err );
-                var statusCode = self.app.findStatusCode(err);
-                self.app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err);
             } else {
                 self.sendResponse(req, res, [tile, 200]);
             }
@@ -100,10 +98,7 @@ LayergroupController.prototype.tileOrLayer = function (req, res) {
             self.app.req2params(req, this);
         },
         function mapController$getTileOrGrid(err) {
-            req.profiler.done('req2params');
+            assert.ifError(err);
-            if ( err ) {
-                throw err;
-            }
             self.tileBackend.getTile(
                 new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
                 req.params, this
@@ -140,18 +135,18 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
         }
     }
 
-    if (err){
+    if (err) {
         // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
         var errMsg = err.message ? ( '' + err.message ) : ( '' + err );
-        var statusCode = this.app.findStatusCode(err);
 
         // Rewrite mapnik parsing errors to start with layer number
         var matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
         if (matches) {
             errMsg = 'style'+matches[2]+': ' + matches[1];
         }
+        err.message = errMsg;
 
-        this.app.sendError(res, { errors: ['' + errMsg] }, statusCode, 'TILE RENDER', err);
+        res.sendError(err, 'TILE RENDER');
         global.statsClient.increment('windshaft.tiles.error');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
     } else {
@@ -189,7 +184,6 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
             self.app.req2params(req, this);
         },
         function(err) {
-            req.profiler.done('req2params');
             assert.ifError(err);
             if (center) {
                 self.previewBackend.getImage(
@@ -206,10 +200,7 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
             req.profiler.add(stats || {});
 
             if (err) {
-                if (!err.error) {
+                res.sendError(err, 'STATIC_MAP');
-                    err.error = err.message;
-                }
-                self.app.sendError(res, {errors: ['' + err] }, self.app.findStatusCode(err), 'STATIC_MAP', err);
             } else {
                 res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format);
                 self.sendResponse(req, res, [image, 200]);
@@ -248,7 +239,7 @@ LayergroupController.prototype.sendResponse = function(req, res, args) {
                 res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
                 self.surrogateKeysCache.tag(res, tablesCacheEntry);
             }
-            self.app.sendResponse(res, args);
+            res.sendResponse(args);
         }
     );
 

lib/cartodb/controllers/map.js

@@ -155,11 +155,10 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         },
         function finish(err, layergroup) {
             if (err) {
-                var statusCode = self.app.findStatusCode(err);
+                res.sendError(err, 'ANONYMOUS LAYERGROUP');
-                self.app.sendError(res, { errors: [ err.message ] }, statusCode, 'ANONYMOUS LAYERGROUP', err);
             } else {
                 res.header('X-Layergroup-Id', layergroup.layergroupid);
-                self.app.sendResponse(res, [layergroup, 200]);
+                res.sendResponse([layergroup, 200]);
             }
         }
     );
@@ -210,8 +209,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
         },
         function finishTemplateInstantiation(err, layergroup) {
             if (err) {
-                var statusCode = self.app.findStatusCode(err);
+                res.sendError(err, 'NAMED MAP LAYERGROUP');
-                self.app.sendError(res, { errors: [ err.message ] }, statusCode, 'NAMED MAP LAYERGROUP', err);
             } else {
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
@@ -219,7 +217,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 res.header('X-Layergroup-Id', layergroup.layergroupid);
                 self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
 
-                self.app.sendResponse(res, [layergroup, 200]);
+                res.sendResponse([layergroup, 200]);
             }
         }
     );

lib/cartodb/controllers/named_maps.js

@@ -60,7 +60,7 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
                     self.surrogateKeysCache.tag(res, tablesCacheEntry);
                 }
             }
-            self.app.sendResponse(res, [resource, 200]);
+            res.sendResponse([resource, 200]);
         }
     );
 };
@@ -90,10 +90,7 @@ NamedMapsController.prototype.tile = function(req, res) {
                 req.profiler.add(stats);
             }
             if (err) {
-                if (!err.error) {
+                res.sendError(err, 'NAMED_MAP_TILE');
-                    err.error = err.message;
-                }
-                self.app.sendError(res, err, self.app.findStatusCode(err), 'NAMED_MAP_TILE', err);
             } else {
                 self.sendResponse(req, res, tile, headers, namedMapProvider);
             }
@@ -182,10 +179,7 @@ NamedMapsController.prototype.staticMap = function(req, res) {
             }
 
             if (err) {
-                if (!err.error) {
+                res.sendError(err, 'STATIC_VIZ_MAP');
-                    err.error = err.message;
-                }
-                self.app.sendError(res, err, self.app.findStatusCode(err), 'STATIC_VIZ_MAP', err);
             } else {
                 self.sendResponse(req, res, image, headers, namedMapProvider);
             }

lib/cartodb/controllers/named_maps_admin.js

@@ -1,18 +1,15 @@
 var step = require('step');
 var assert = require('assert');
-var _ = require('underscore');
 var templateName = require('../backends/template_maps').templateName;
 var cors = require('../middleware/cors');
 
 
 /**
- * @param app
  * @param {TemplateMaps} templateMaps
  * @param {AuthApi} authApi
  * @constructor
  */
-function NamedMapsAdminController(app, templateMaps, authApi) {
+function NamedMapsAdminController(templateMaps, authApi) {
-    this.app = app;
     this.templateMaps = templateMaps;
     this.authApi = authApi;
 }
@@ -48,7 +45,7 @@ NamedMapsAdminController.prototype.create = function(req, res) {
             assert.ifError(err);
             return { template_id: tpl_id };
         },
-        finishFn(self.app, res, 'POST TEMPLATE')
+        finishFn(res, 'POST TEMPLATE')
     );
 };
 
@@ -76,7 +73,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
 
             return { template_id: tpl_id };
         },
-        finishFn(self.app, res, 'PUT TEMPLATE')
+        finishFn(res, 'PUT TEMPLATE')
     );
 };
 
@@ -112,7 +109,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
             delete tpl_val.auth_id;
             return { template: tpl_val };
         },
-        finishFn(self.app, res, 'GET TEMPLATE')
+        finishFn(res, 'GET TEMPLATE')
     );
 };
 
@@ -140,7 +137,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
             assert.ifError(err);
             return { status: 'ok' };
         },
-        finishFn(self.app, res, 'DELETE TEMPLATE', ['', 204])
+        finishFn(res, 'DELETE TEMPLATE', ['', 204])
     );
 };
 
@@ -166,22 +163,16 @@ NamedMapsAdminController.prototype.list = function(req, res) {
             assert.ifError(err);
             return { template_ids: tpl_ids };
         },
-        finishFn(self.app, res, 'GET TEMPLATE LIST')
+        finishFn(res, 'GET TEMPLATE LIST')
     );
 };
 
-function finishFn(app, res, description, okResponse) {
+function finishFn(res, description, okResponse) {
     return function finish(err, response){
-        var statusCode = 200;
         if (err) {
-            statusCode = 400;
+            res.sendError(err, description);
-            response = { errors: ['' + err] };
-            if ( ! _.isUndefined(err.http_status) ) {
-                statusCode = err.http_status;
-            }
-            app.sendError(res, response, statusCode, description, err);
         } else {
-            app.sendResponse(res, okResponse || [response, statusCode]);
+            res.sendResponse(okResponse || [response, 200]);
         }
     };
 }

lib/cartodb/monitoring/map-config.xml

@@ -1,4 +0,0 @@
-<Map
-    background-color="#c33"
-    srs="+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs">
-</Map>

lib/cartodb/server.js

@@ -13,6 +13,10 @@ var NamedMapsCacheEntry = require('./cache/model/named_maps_entry');
 var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
 var FastlyCacheBackend = require('./cache/backend/fastly');
 
+var StatsClient = require('./stats/client');
+var Profiler = require('./stats/profiler_proxy');
+var RendererStatsReporter = require('./stats/reporter/renderer');
+
 var windshaft = require('windshaft');
 var mapnik = windshaft.mapnik;
 
@@ -46,7 +50,7 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin
 
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
-    global.statsClient = windshaft.stats.Client.getInstance(serverOptions.statsd);
+    global.statsClient = StatsClient.getInstance(serverOptions.statsd);
 
     var redisPool = new RedisPool(_.defaults(global.environment.redis, {
         name: 'windshaft:server',
@@ -76,22 +80,7 @@ module.exports = function(serverOptions) {
         max_user_templates: global.environment.maxUserTemplates
     });
 
-    var surrogateKeysCacheBackends = [];
+    var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions));
-
-    if (serverOptions.varnish_purge_enabled) {
-        surrogateKeysCacheBackends.push(
-            new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
-        );
-    }
-
-    if (serverOptions.fastly &&
-        !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
-        surrogateKeysCacheBackends.push(
-            new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
-        );
-    }
-
-    var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends);
 
     function invalidateNamedMap (owner, templateName) {
         var startTime = Date.now();
@@ -156,6 +145,8 @@ module.exports = function(serverOptions) {
         statsInterval: 60000 // reports stats every milliseconds defined here
     });
     var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
+    var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
+    rendererStatsReporter.start();
 
     var attributesBackend = new windshaft.backend.Attributes(mapStore);
     var previewBackend = new windshaft.backend.Preview(rendererCache);
@@ -173,16 +164,6 @@ module.exports = function(serverOptions) {
 
     var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);
 
-    app.findStatusCode = function(err) {
-        var statusCode;
-        if ( err.http_status ) {
-            statusCode = err.http_status;
-        } else {
-            statusCode = statusFromErrorMessage('' + err);
-        }
-        return statusCode;
-    };
-
     var TablesExtentApi = require('./api/tables_extent_api');
     var tablesExtentApi = new TablesExtentApi(pgQueryRunner);
 
@@ -228,7 +209,7 @@ module.exports = function(serverOptions) {
         tablesExtentApi
     ).register(app);
 
-    new controller.NamedMapsAdmin(app, templateMaps, authApi).register(app);
+    new controller.NamedMapsAdmin(templateMaps, authApi).register(app);
 
     new controller.ServerInfo().register(app);
 
@@ -236,81 +217,6 @@ module.exports = function(serverOptions) {
      * END Routing
      ******************************************************************************************************************/
 
-        // temporary measure until we upgrade to newer version expressjs so we can check err.status
-    app.use(function(err, req, res, next) {
-        if (err) {
-            if (err.name === 'SyntaxError') {
-                app.sendError(res, { errors: [err.name + ': ' + err.message] }, 400, 'JSON', err);
-            } else {
-                next(err);
-            }
-        } else {
-            next();
-        }
-    });
-
-    app.sendResponse = function(res, args) {
-        var req = res.req;
-
-        if (global.environment && global.environment.api_hostname) {
-            res.header('X-Served-By-Host', global.environment.api_hostname);
-        }
-
-        if (req && req.params && req.params.dbhost) {
-            res.header('X-Served-By-DB-Host', req.params.dbhost);
-        }
-
-        if ( req && req.profiler ) {
-            res.header('X-Tiler-Profiler', req.profiler.toJSONString());
-        }
-
-//        res.send(body|status[, headers|status[, status]])
-        res.send.apply(res, args);
-
-        if ( req && req.profiler ) {
-            try {
-                // May throw due to dns, see
-                // See http://github.com/CartoDB/Windshaft/issues/166
-                req.profiler.sendStats();
-            } catch (err) {
-                console.error("error sending profiling stats: " + err);
-            }
-        }
-    };
-
-    app.sendError = function(res, err, statusCode, label, tolog) {
-        res._windshaftStatusCode = statusCode;
-
-        var olabel = '[';
-        if ( label ) {
-            olabel += label + ' ';
-        }
-        olabel += 'ERROR]';
-        if ( ! tolog ) {
-            tolog = err;
-        }
-        var log_msg = olabel + " -- " + statusCode + ": " + tolog;
-        //if ( tolog.stack ) log_msg += "\n" + tolog.stack;
-        console.error(log_msg); // use console.log for statusCode != 500 ?
-        // If a callback was requested, force status to 200
-        if ( res.req ) {
-            // NOTE: res.req can be undefined when we fake a call to
-            //       ourself from POST to /layergroup
-            if ( res.req.query.callback ) {
-                statusCode = 200;
-            }
-        }
-        // Strip connection info, if any
-        // See https://github.com/CartoDB/Windshaft/issues/173
-        err = JSON.stringify(err);
-        err = err.replace(/Connection string: '[^']*'\\n/, '');
-        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
-        err = err.replace(/is the server.*encountered/im, 'encountered');
-        err = JSON.parse(err);
-
-        app.sendResponse(res, [err, statusCode]);
-    };
-
     // jshint maxcomplexity:10
     /**
      * Whitelist input and get database name & default geometry type from
@@ -342,6 +248,7 @@ module.exports = function(serverOptions) {
                         _.extend(req.query, JSON.parse(result));
                         app.req2params(req, callback);
                     } catch (err) {
+                        req.profiler.done('req2params');
                         callback(new Error('Error parsing lzma as JSON: ' + err));
                     }
                 }
@@ -374,6 +281,7 @@ module.exports = function(serverOptions) {
                         'Cannot use map signature of user "' + req.params.signer + '" on db of user "' + user + '"'
                     );
                     err.http_status = 403;
+                    req.profiler.done('req2params');
                     callback(err);
                     return;
                 }
@@ -414,6 +322,7 @@ module.exports = function(serverOptions) {
             },
             function finishSetup(err) {
                 if ( err ) {
+                    req.profiler.done('req2params');
                     return callback(err, req);
                 }
 
@@ -426,10 +335,12 @@ module.exports = function(serverOptions) {
                     dbport: global.environment.postgres.port
                 });
 
+                req.profiler.done('req2params');
                 callback(null, req);
             }
         );
     };
+    // jshint maxcomplexity:6
 
     return app;
 };
@@ -472,14 +383,85 @@ function bootstrap(opts) {
 
     app.use(function bootstrap$prepareRequestResponse(req, res, next) {
         req.context = req.context || {};
-        req.profiler = new windshaft.stats.Profiler({
+        req.profiler = new Profiler({
             statsd_client: global.statsClient,
             profile: opts.useProfiler
         });
+
         res.removeHeader('x-powered-by');
+
+        res.sendResponse = function(args) {
+            if (global.environment && global.environment.api_hostname) {
+                res.header('X-Served-By-Host', global.environment.api_hostname);
+            }
+
+            if (req.params && req.params.dbhost) {
+                res.header('X-Served-By-DB-Host', req.params.dbhost);
+            }
+
+            if (req.profiler) {
+                res.header('X-Tiler-Profiler', req.profiler.toJSONString());
+            }
+
+            res.send.apply(res, args);
+
+            if (req.profiler ) {
+                try {
+                    // May throw due to dns, see
+                    // See http://github.com/CartoDB/Windshaft/issues/166
+                    req.profiler.sendStats();
+                } catch (err) {
+                    console.error("error sending profiling stats: " + err);
+                }
+            }
+        };
+        res.sendError = function(err, label) {
+
+            label = label || 'UNKNOWN';
+
+            var statusCode = findStatusCode(err);
+
+            // use console.log for statusCode != 500 ?
+            if (statusCode >= 500) {
+                console.error('[%s ERROR] -- %d: %s', label, statusCode, err);
+            } else {
+                console.warn('[%s WARN] -- %d: %s', label, statusCode, err);
+            }
+
+            // If a callback was requested, force status to 200
+            if (req.query.callback) {
+                statusCode = 200;
+            }
+
+            // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+            var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+            // Strip connection info, if any
+            message = message
+                // See https://github.com/CartoDB/Windshaft/issues/173
+                .replace(/Connection string: '[^']*'\\n/, '')
+                // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
+                .replace(/is the server.*encountered/im, 'encountered');
+
+            var errorResponseBody = { errors: [message] };
+
+            res.sendResponse([errorResponseBody, statusCode]);
+        };
         next();
     });
 
+    // temporary measure until we upgrade to newer version expressjs so we can check err.status
+    app.use(function(err, req, res, next) {
+        if (err) {
+            if (err.name === 'SyntaxError') {
+                res.send({ errors: [err.name + ': ' + err.message] }, 400);
+            } else {
+                next(err);
+            }
+        } else {
+            next();
+        }
+    });
+
     setupLogger(app, opts);
 
     return app;
@@ -505,6 +487,36 @@ function setupLogger(app, opts) {
     }
 }
 
+function surrogateKeysCacheBackends(serverOptions) {
+    var cacheBackends = [];
+
+    if (serverOptions.varnish_purge_enabled) {
+        cacheBackends.push(
+            new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
+        );
+    }
+
+    if (serverOptions.fastly &&
+        !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
+        cacheBackends.push(
+            new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
+        );
+    }
+
+    return cacheBackends;
+}
+
+function findStatusCode(err) {
+    var statusCode;
+    if ( err.http_status ) {
+        statusCode = err.http_status;
+    } else {
+        statusCode = statusFromErrorMessage('' + err);
+    }
+    return statusCode;
+}
+module.exports.findStatusCode = findStatusCode;
+
 function statusFromErrorMessage(errMsg) {
     // Find an appropriate statusCode based on message
     var statusCode = 400;

lib/cartodb/stats/client.js

@@ -0,0 +1,74 @@
+var _ = require('underscore');
+var debug = require('debug')('windshaft:stats_client');
+var StatsD = require('node-statsd').StatsD;
+
+module.exports = {
+    /**
+     * Returns an StatsD instance or an stub object that replicates the StatsD public interface so there is no need to
+     * keep checking if the stats_client is instantiated or not.
+     *
+     * The first call to this method implies all future calls will use the config specified in the very first call.
+     *
+     * TODO: It's far from ideal to use make this a singleton, improvement desired.
+     * We proceed this way to be able to use StatsD from several places sharing one single StatsD instance.
+     *
+     * @param config Configuration for StatsD, if undefined it will return an stub
+     * @returns {StatsD|Object}
+     */
+    getInstance: function(config) {
+
+        if (!this.instance) {
+
+            var instance;
+
+            if (config) {
+                instance = new StatsD(config);
+                instance.last_error = { msg: '', count: 0 };
+                instance.socket.on('error', function (err) {
+                    var last_err = instance.last_error;
+                    var last_msg = last_err.msg;
+                    var this_msg = '' + err;
+                    if (this_msg !== last_msg) {
+                        debug("statsd client socket error: " + err);
+                        instance.last_error.count = 1;
+                        instance.last_error.msg = this_msg;
+                    } else {
+                        ++last_err.count;
+                        if (!last_err.interval) {
+                            instance.last_error.interval = setInterval(function () {
+                                var count = instance.last_error.count;
+                                if (count > 1) {
+                                    debug("last statsd client socket error repeated " + count + " times");
+                                    instance.last_error.count = 1;
+                                    //console.log("Clearing interval");
+                                    clearInterval(instance.last_error.interval);
+                                    instance.last_error.interval = null;
+                                }
+                            }, 1000);
+                        }
+                    }
+                });
+            } else {
+                var stubFunc = function (stat, value, sampleRate, callback) {
+                    if (_.isFunction(callback)) {
+                        callback(null, 0);
+                    }
+                };
+                instance = {
+                    timing: stubFunc,
+                    increment: stubFunc,
+                    decrement: stubFunc,
+                    gauge: stubFunc,
+                    unique: stubFunc,
+                    set: stubFunc,
+                    sendAll: stubFunc,
+                    send: stubFunc
+                };
+            }
+
+            this.instance = instance;
+        }
+
+        return this.instance;
+    }
+};

lib/cartodb/stats/profiler_proxy.js

@@ -0,0 +1,53 @@
+var Profiler = require('step-profiler');
+
+/**
+ * Proxy to encapsulate node-step-profiler module so there is no need to check if there is an instance
+ */
+function ProfilerProxy(opts) {
+    this.profile = !!opts.profile;
+
+    this.profiler = null;
+    if (!!opts.profile) {
+        this.profiler = new Profiler({statsd_client: opts.statsd_client});
+    }
+}
+
+ProfilerProxy.prototype.done = function(what) {
+    if (this.profile) {
+        this.profiler.done(what);
+    }
+};
+
+ProfilerProxy.prototype.end = function() {
+    if (this.profile) {
+        this.profiler.end();
+    }
+};
+
+ProfilerProxy.prototype.start = function(what) {
+    if (this.profile) {
+        this.profiler.start(what);
+    }
+};
+
+ProfilerProxy.prototype.add = function(what) {
+    if (this.profile) {
+        this.profiler.add(what || {});
+    }
+};
+
+ProfilerProxy.prototype.sendStats = function() {
+    if (this.profile) {
+        this.profiler.sendStats();
+    }
+};
+
+ProfilerProxy.prototype.toString = function() {
+    return this.profile ? this.profiler.toString() : "";
+};
+
+ProfilerProxy.prototype.toJSONString = function() {
+    return this.profile ? this.profiler.toJSONString() : "{}";
+};
+
+module.exports = ProfilerProxy;

lib/cartodb/stats/reporter/renderer.js

@@ -0,0 +1,83 @@
+//  - Reports stats about:
+//    * Total number of renderers
+//    * For mapnik renderers:
+//      - the mapnik-pool status: count, unused and waiting
+//      - the internally cached objects: png and grid
+
+var _ = require('underscore');
+
+function RendererStatsReporter(rendererCache, statsInterval) {
+    this.rendererCache = rendererCache;
+    this.statsInterval = statsInterval || 6e4;
+    this.renderersStatsIntervalId = null;
+}
+
+module.exports = RendererStatsReporter;
+
+RendererStatsReporter.prototype.start = function() {
+    var self = this;
+    this.renderersStatsIntervalId = setInterval(function() {
+        var rendererCacheEntries = self.rendererCache.renderers;
+
+        if (!rendererCacheEntries) {
+            return null;
+        }
+
+        global.statsClient.gauge('windshaft.rendercache.count', _.keys(rendererCacheEntries).length);
+
+        var renderersStats = _.reduce(rendererCacheEntries, function(_rendererStats, cacheEntry) {
+                var stats = cacheEntry.renderer && cacheEntry.renderer.getStats && cacheEntry.renderer.getStats();
+                if (!stats) {
+                    return _rendererStats;
+                }
+
+                _rendererStats.pool.count += stats.pool.count;
+                _rendererStats.pool.unused += stats.pool.unused;
+                _rendererStats.pool.waiting += stats.pool.waiting;
+
+                _rendererStats.cache.grid += stats.cache.grid;
+                _rendererStats.cache.png += stats.cache.png;
+
+                return _rendererStats;
+            },
+            {
+                pool: {
+                    count: 0,
+                    unused: 0,
+                    waiting: 0
+                },
+                cache: {
+                    png: 0,
+                    grid: 0
+                }
+            }
+        );
+
+        global.statsClient.gauge('windshaft.mapnik-cache.png', renderersStats.cache.png);
+        global.statsClient.gauge('windshaft.mapnik-cache.grid', renderersStats.cache.grid);
+
+        global.statsClient.gauge('windshaft.mapnik-pool.count', renderersStats.pool.count);
+        global.statsClient.gauge('windshaft.mapnik-pool.unused', renderersStats.pool.unused);
+        global.statsClient.gauge('windshaft.mapnik-pool.waiting', renderersStats.pool.waiting);
+    }, this.statsInterval);
+
+    this.rendererCache.on('err', rendererCacheErrorListener);
+    this.rendererCache.on('gc', gcTimingListener);
+};
+
+function rendererCacheErrorListener() {
+    global.statsClient.increment('windshaft.rendercache.error');
+}
+
+function gcTimingListener(gcTime) {
+    console.log('gctime');
+    global.statsClient.timing('windshaft.rendercache.gc', gcTime);
+}
+
+RendererStatsReporter.prototype.stop = function() {
+    this.rendererCache.removeListener('err', rendererCacheErrorListener);
+    this.rendererCache.removeListener('gc', gcTimingListener);
+
+    clearInterval(this.renderersStatsIntervalId);
+    this.renderersStatsIntervalId = null;
+};

npm-shrinkwrap.json

@@ -66,6 +66,18 @@
         }
       }
     },
+    "debug": {
+      "version": "2.2.0",
+      "from": "debug@~2.2.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
+      "dependencies": {
+        "ms": {
+          "version": "0.7.1",
+          "from": "ms@0.7.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+        }
+      }
+    },
     "dot": {
       "version": "1.0.3",
       "from": "dot@~1.0.2",
@@ -140,9 +152,9 @@
                       "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
                     },
                     "process-nextick-args": {
-                      "version": "1.0.2",
+                      "version": "1.0.3",
                       "from": "process-nextick-args@~1.0.0",
-                      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.2.tgz"
+                      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.3.tgz"
                     },
                     "string_decoder": {
                       "version": "0.10.31",
@@ -255,9 +267,9 @@
               "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.0.tgz",
               "dependencies": {
                 "hoek": {
-                  "version": "2.14.0",
+                  "version": "2.15.0",
                   "from": "hoek@2.x.x",
-                  "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.14.0.tgz"
+                  "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.15.0.tgz"
                 },
                 "boom": {
                   "version": "2.8.0",
@@ -265,9 +277,9 @@
                   "resolved": "https://registry.npmjs.org/boom/-/boom-2.8.0.tgz"
                 },
                 "cryptiles": {
-                  "version": "2.0.4",
+                  "version": "2.0.5",
                   "from": "cryptiles@2.x.x",
-                  "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.4.tgz"
+                  "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
                 },
                 "sntp": {
                   "version": "1.0.9",
@@ -309,9 +321,9 @@
               "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz",
               "dependencies": {
                 "bluebird": {
-                  "version": "2.9.34",
+                  "version": "2.10.0",
                   "from": "bluebird@^2.9.30",
-                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.34.tgz"
+                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.10.0.tgz"
                 },
                 "chalk": {
                   "version": "1.1.1",
@@ -470,6 +482,11 @@
       "from": "lzma@~1.3.7",
       "resolved": "https://registry.npmjs.org/lzma/-/lzma-1.3.7.tgz"
     },
+    "node-statsd": {
+      "version": "0.0.7",
+      "from": "node-statsd@~0.0.7",
+      "resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.0.7.tgz"
+    },
     "queue-async": {
       "version": "1.0.7",
       "from": "queue-async@~1.0.7",
@@ -518,6 +535,11 @@
       "from": "step@~0.0.5",
       "resolved": "https://registry.npmjs.org/step/-/step-0.0.6.tgz"
     },
+    "step-profiler": {
+      "version": "0.2.1",
+      "from": "step-profiler@~0.2.1",
+      "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.2.1.tgz"
+    },
     "underscore": {
       "version": "1.6.0",
       "from": "underscore@~1.6.0",
@@ -2287,18 +2309,6 @@
             }
           }
         },
-        "debug": {
-          "version": "2.2.0",
-          "from": "debug@~2.2.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
-          "dependencies": {
-            "ms": {
-              "version": "0.7.1",
-              "from": "ms@0.7.1",
-              "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-            }
-          }
-        },
         "tilelive": {
           "version": "4.5.3",
           "from": "tilelive@~4.5.3",
@@ -3175,11 +3185,6 @@
             }
           }
         },
-        "step-profiler": {
-          "version": "0.2.1",
-          "from": "step-profiler@~0.2.1",
-          "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.2.1.tgz"
-        },
         "torque.js": {
           "version": "2.11.0",
           "from": "torque.js@~2.11.0",
@@ -3368,11 +3373,6 @@
           "version": "1.0.2",
           "from": "sphericalmercator@1.0.2",
           "resolved": "https://registry.npmjs.org/sphericalmercator/-/sphericalmercator-1.0.2.tgz"
-        },
-        "node-statsd": {
-          "version": "0.0.7",
-          "from": "node-statsd@~0.0.7",
-          "resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.0.7.tgz"
         }
       }
     }

package.json

@@ -23,6 +23,9 @@
     ],
     "dependencies": {
         "express": "~2.5.11",
+        "debug": "~2.2.0",
+        "step-profiler": "~0.2.1",
+        "node-statsd": "~0.0.7",
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
         "windshaft": "https://github.com/CartoDB/Windshaft/tarball/backend-foundations-mvt",

test/acceptance/named_static_maps.js

@@ -163,8 +163,10 @@ describe('named static maps', function() {
         var nonexistentName = 'nonexistent';
         getStaticMap(nonexistentName, { status: 404 }, function(err, res) {
             assert.ok(!err);
-            var parsed = JSON.parse(res.body);
+            assert.deepEqual(
-            assert.equal(parsed.error, "Template '" + nonexistentName + "' of user '" + username + "' not found");
+                JSON.parse(res.body),
+                { errors: ["Template '" + nonexistentName + "' of user '" + username + "' not found"] }
+            );
             done();
         });
     });
@@ -172,8 +174,7 @@ describe('named static maps', function() {
     it('should return 403 if not properly authorized', function(done) {
         getStaticMap(tokenAuthTemplateName, { status: 403 }, function(err, res) {
             assert.ok(!err);
-            var parsed = JSON.parse(res.body);
+            assert.deepEqual(JSON.parse(res.body), { errors: ['Unauthorized template instantiation'] });
-            assert.equal(parsed.error, 'Unauthorized template instantiation');
             done();
         });
     });

test/unit/cartodb/ported/profiler.test.js

@@ -0,0 +1,17 @@
+require('../../../support/test_helper');
+
+var assert = require('assert');
+var ProfilerProxy = require('../../../../lib/cartodb/stats/profiler_proxy');
+
+describe('profiler', function() {
+
+    it('Profiler is null in ProfilerProxy when profiling is not enabled', function() {
+        var profilerProxy = new ProfilerProxy({profile: false});
+        assert.equal(profilerProxy.profiler, null);
+    });
+
+    it('Profiler is NOT null in ProfilerProxy when profiling is enabled', function() {
+        var profilerProxy = new ProfilerProxy({profile: true});
+        assert.notEqual(profilerProxy.profiler, null);
+    });
+});

test/unit/cartodb/ported/stats_client.test.js

@@ -0,0 +1,40 @@
+require('../../../support/test_helper');
+
+var assert = require('assert');
+
+var StatsClient = require('../../../../lib/cartodb/stats/client');
+
+describe('stats client', function() {
+    var statsInstance;
+
+    before(function() {
+        statsInstance = StatsClient.instance;
+        StatsClient.instance = null;
+    });
+
+    after(function() {
+        StatsClient.instance = statsInstance;
+    });
+
+    it('reports errors when they repeat', function(done) {
+        var WADUS_ERROR = 'wadus_error';
+        var statsClient = StatsClient.getInstance({ host: '127.0.0.1', port: 8033 });
+
+        statsClient.socket.emit('error', 'other_error');
+        assert.ok(statsClient.last_error);
+        assert.equal(statsClient.last_error.msg, 'other_error');
+        assert.ok(!statsClient.last_error.interval);
+
+        statsClient.socket.emit('error', WADUS_ERROR);
+        assert.ok(statsClient.last_error);
+        assert.equal(statsClient.last_error.msg, WADUS_ERROR);
+        assert.ok(!statsClient.last_error.interval);
+
+        statsClient.socket.emit('error', WADUS_ERROR);
+        assert.ok(statsClient.last_error);
+        assert.equal(statsClient.last_error.msg, WADUS_ERROR);
+        assert.ok(statsClient.last_error.interval);
+
+        done();
+    });
+});

test/unit/cartodb/ported/tile_stats.test.js

@@ -3,7 +3,7 @@ require('../../../support/test_helper.js');
 var assert = require('assert');
 var cartodbServer = require('../../../../lib/cartodb/server');
 var serverOptions = require('../../../../lib/cartodb/server_options');
-var StatsClient = require('windshaft').stats.Client;
+var StatsClient = require('../../../../lib/cartodb/stats/client');
 
 var LayergroupController = require('../../../../lib/cartodb/controllers/layergroup');
 
@@ -28,17 +28,17 @@ describe('tile stats', function() {
             }
         });
 
-        var ws = cartodbServer(serverOptions);
+        var layergroupController = new LayergroupController(cartodbServer(serverOptions));
-        ws.sendError = function(){};
-
-        var layergroupController = new LayergroupController(ws, null);
 
         var reqMock = {
             params: {
                 format: invalidFormat
             }
         };
-        layergroupController.finalizeGetTileOrGrid('Unsupported format png2', reqMock, {}, null, null);
+        var resMock = {
+            sendError: function() {}
+        };
+        layergroupController.finalizeGetTileOrGrid('Unsupported format png2', reqMock, resMock, null, null);
 
         assert.ok(formatMatched, 'Format was never matched in increment method');
         assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method');
@@ -60,13 +60,13 @@ describe('tile stats', function() {
                 format: validFormat
             }
         };
+        var resMock = {
+            sendError: function() {}
+        };
 
-        var ws = cartodbServer(serverOptions);
+        var layergroupController = new LayergroupController(cartodbServer(serverOptions));
-        ws.sendError = function(){};
 
-        var layergroupController = new LayergroupController(ws, null);
+        layergroupController.finalizeGetTileOrGrid('Another error happened', reqMock, resMock, null, null);
-
-        layergroupController.finalizeGetTileOrGrid('Another error happened', reqMock, {}, null, null);
 
         assert.ok(formatMatched, 'Format was never matched in increment method');
         assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method');

test/unit/cartodb/ported/windshaft_server.test.js

@@ -41,16 +41,16 @@ describe('windshaft', function() {
     });
 
     it('different formats for postgis plugin error returns 400 as status code', function() {
-        var ws = cartodbServer(serverOptions);
+
         var expectedStatusCode = 400;
         assert.equal(
-            ws.findStatusCode("Postgis Plugin: ERROR:  column \"missing\" does not exist\n"),
+            cartodbServer.findStatusCode("Postgis Plugin: ERROR:  column \"missing\" does not exist\n"),
             expectedStatusCode,
             "Error status code for single line does not match"
         );
 
         assert.equal(
-            ws.findStatusCode("Postgis Plugin: PSQL error:\nERROR:  column \"missing\" does not exist\n"),
+            cartodbServer.findStatusCode("Postgis Plugin: PSQL error:\nERROR:  column \"missing\" does not exist\n"),
             expectedStatusCode,
             "Error status code for multiline/PSQL does not match"
         );

test/unit/cartodb/req2params.test.js

@@ -18,14 +18,17 @@ suite('req2params', function() {
       assert.ok(_.isFunction(server.req2params));
     });
 
-    function addContext(req) {
+    function prepareRequest(req) {
+        req.profiler = {
+            done: function() {}
+        };
         req.context = { user: 'localhost' };
         return req;
     }
 
     test('cleans up request', function(done){
       var req = {headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}};
-      server.req2params(addContext(req), function(err, req) {
+      server.req2params(prepareRequest(req), function(err, req) {
           if ( err ) { done(err); return; }
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
@@ -39,7 +42,7 @@ suite('req2params', function() {
 
     test('sets dbname from redis metadata', function(done){
       var req = {headers: { host:'localhost' }, query: {} };
-      server.req2params(addContext(req), function(err, req) {
+      server.req2params(prepareRequest(req), function(err, req) {
           if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
@@ -54,7 +57,7 @@ suite('req2params', function() {
 
     test('sets also dbuser for authenticated requests', function(done){
       var req = {headers: { host:'localhost' }, query: {map_key: '1234'} };
-      server.req2params(addContext(req), function(err, req) {
+      server.req2params(prepareRequest(req), function(err, req) {
           if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
@@ -63,8 +66,16 @@ suite('req2params', function() {
           assert.ok(!req.params.hasOwnProperty('interactivity'), 'request params do not have interactivity');
           assert.equal(req.params.dbname, test_database);
           assert.equal(req.params.dbuser, test_user);
- 
+
-          server.req2params(addContext({headers: { host:'localhost' }, query: {map_key: '1235'} }), function(err, req) {
+          req = {
+              headers: {
+                  host:'localhost'
+              },
+              query: {
+                  map_key: '1235'
+              }
+          };
+          server.req2params(prepareRequest(req), function(err, req) {
               // wrong key resets params to no user
               assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
               done();
@@ -90,7 +101,7 @@ suite('req2params', function() {
                     lzma: data
                 }
             };
-            server.req2params(addContext(req), function(err, req) {
+            server.req2params(prepareRequest(req), function(err, req) {
                 if ( err ) {
                     return done(err);
                 }