diff --git a/NEWS.md b/NEWS.md index 6cf0f08b..06366d5f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,6 +6,7 @@ New features: * Add optional support for rollbar (#137) * Add '/version' endpoint (#138) * Add profiler support (#142) + * Add statsd support (#133) Enhancements: diff --git a/app/controllers/app.js b/app/controllers/app.js index 37d1dbec..6fe84a93 100755 --- a/app/controllers/app.js +++ b/app/controllers/app.js @@ -24,10 +24,12 @@ var express = require('express') , Step = require('step') , crypto = require('crypto') , fs = require('fs') + , os = require('os') , zlib = require('zlib') , util = require('util') , spawn = require('child_process').spawn , Profiler = require('step-profiler') + , StatsD = require('node-statsd').StatsD , Meta = require('cartodb-redis')({ host: global.settings.redis_host, port: global.settings.redis_port @@ -86,10 +88,50 @@ if ( global.log4js ) { app.use(express.logger(loggerOpts)); } +// Initialize statsD client if requested +var statsd_client; +if ( global.settings.statsd ) { + + // Perform keyword substitution in statsd + if ( global.settings.statsd.prefix ) { + var host_token = os.hostname().split('.').reverse().join('.'); + global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, host_token); + } + + statsd_client = new StatsD(global.settings.statsd); + statsd_client.last_error = { msg:'', count:0 }; + statsd_client.socket.on('error', function(err) { + var last_err = statsd_client.last_error; + var last_msg = last_err.msg; + var this_msg = ''+err; + if ( this_msg != last_msg ) { + console.error("statsd client socket error: " + err); + statsd_client.last_error.count = 1; + statsd_client.last_error.msg = this_msg; + } else { + ++last_err.count; + if ( ! last_err.interval ) { + //console.log("Installing interval"); + statsd_client.last_error.interval = setInterval(function() { + var count = statsd_client.last_error.count + if ( count > 1 ) { + console.error("last statsd client socket error repeated " + count + " times"); + statsd_client.last_error.count = 1; + //console.log("Clearing interval"); + clearInterval(statsd_client.last_error.interval); + statsd_client.last_error.interval = null; + } + }, 1000); + } + } + }); +} + + // Use step-profiler if ( global.settings.useProfiler ) { app.use(function(req, res, next) { - req.profiler = new Profiler({}); + req.profiler = new Profiler({statsd_client:statsd_client}); next(); }); } @@ -235,7 +277,7 @@ function handleQuery(req, res) { var cdbuser = cdbReq.userByReq(req); - if ( req.profiler ) req.profiler.start('init'); + if ( req.profiler ) req.profiler.done('init'); // 1. Get database from redis via the username stored in the host header subdomain // 2. Run the request through OAuth to get R/W user id if signed @@ -437,10 +479,15 @@ function handleQuery(req, res) { if ( req.profiler ) { req.profiler.sendStats(); // TODO: do on nextTick ? } + if (statsd_client) { + if ( err ) statsd_client.increment('sqlapi.query.error'); + else statsd_client.increment('sqlapi.query.success'); + } } ); } catch (err) { handleException(err, res); + if (statsd_client) statsd_client.increment('sqlapi.query.error'); } } diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 6ffb116d..bc0365c0 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -43,3 +43,11 @@ module.exports.tableCacheMax = 8192; module.exports.tableCacheMaxAge = 1000*60*10; // Temporary directory, make sure it is writable by server user module.exports.tmpDir = '/tmp'; +// Optional statsd support +module.exports.statsd = { + host: 'localhost', + port: 8125, + prefix: 'dev.:host.', + cacheDns: true + // support all allowed node-statsd options +} diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 8e616fb5..772b5584 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -52,3 +52,11 @@ module.exports.rollbar = { handler: 'inline' } } +// Optional statsd support +module.exports.statsd = { + host: 'localhost', + port: 8125, + prefix: 'dev.:host.', + cacheDns: true + // support all allowed node-statsd options +} diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index bd96518c..07332b72 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -52,3 +52,11 @@ module.exports.rollbar = { handler: 'inline' } } +// Optional statsd support +module.exports.statsd = { + host: 'localhost', + port: 8125, + prefix: 'dev.:host.', + cacheDns: true + // support all allowed node-statsd options +} diff --git a/config/environments/test.js.example b/config/environments/test.js.example index e4c92118..e87969f3 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -41,3 +41,11 @@ module.exports.tableCacheMax = 8192; module.exports.tableCacheMaxAge = 1000*60*10; // Temporary directory, make sure it is writable by server user module.exports.tmpDir = '/tmp'; +// Optional statsd support +module.exports.statsd = { + host: 'localhost', + port: 8125, + prefix: 'dev.:host.', + cacheDns: true + // support all allowed node-statsd options +} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 33122bbb..8061fdc4 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -132,6 +132,9 @@ "version": "0.0.1", "from": "git://github.com/CartoDB/node-step-profiler.git#0.0.1" }, + "node-statsd": { + "version": "0.0.7" + }, "redis": { "version": "0.7.1" }, diff --git a/package.json b/package.json index 4072d9bb..ac9e902f 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "lru-cache":"~2.2.2", "log4js": "~0.6.10", "rollbar": "~0.3.1", - "step-profiler": "git://github.com/CartoDB/node-step-profiler.git#0.0.1" + "step-profiler": "git://github.com/CartoDB/node-step-profiler.git#0.0.1", + "node-statsd": "~0.0.7" }, "devDependencies": { "redis": "0.7.1",