diff --git a/NEWS.md b/NEWS.md index e3446ed8..6e8bca0f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,16 @@ #Changelog -## 1.43.2 +## 1.44.1 Released 2017-mm-dd +## 1.44.0 +Released 2017-03-30 + +Announcements: + * Active GC interval for Node.js >=v6. + + ## 1.43.1 Released 2017-01-16 diff --git a/app.js b/app.js index ad973a5b..778eb1de 100755 --- a/app.js +++ b/app.js @@ -11,6 +11,7 @@ */ var fs = require('fs'); var path = require('path'); +var os = require('os'); var argv = require('yargs') .usage('Usage: $0 [options]') @@ -80,7 +81,17 @@ if ( ! global.settings.base_url ) { var version = require("./package").version; -var server = require('./app/server')(); +var StatsClient = require('./app/stats/client'); +if (global.settings.statsd) { + // Perform keyword substitution in statsd + if (global.settings.statsd.prefix) { + var hostToken = os.hostname().split('.').reverse().join('.'); + global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, hostToken); + } +} +var statsClient = StatsClient.getInstance(global.settings.statsd); + +var server = require('./app/server')(statsClient); var listener = server.listen(global.settings.node_port, global.settings.node_host); listener.on('listening', function() { console.info("Using Node.js %s", process.version); @@ -119,3 +130,28 @@ process.on('SIGTERM', function () { process.exit(0); }); }); + +function isGteMinVersion(version, minVersion) { + var versionMatch = /[a-z]?([0-9]*)/.exec(version); + if (versionMatch) { + var majorVersion = parseInt(versionMatch[1], 10); + if (Number.isFinite(majorVersion)) { + return majorVersion >= minVersion; + } + } + return false; +} + +if (global.gc && isGteMinVersion(process.version, 6)) { + var gcInterval = Number.isFinite(global.settings.gc_interval) ? + global.settings.gc_interval : + 10000; + + if (gcInterval > 0) { + setInterval(function gcForcedCycle() { + var start = Date.now(); + global.gc(); + statsClient.timing('sqlapi.gc', Date.now() - start); + }, gcInterval); + } +} diff --git a/app/server.js b/app/server.js index 04b7bc92..2eb88d79 100644 --- a/app/server.js +++ b/app/server.js @@ -16,9 +16,7 @@ var express = require('express'); var bodyParser = require('./middlewares/body-parser'); -var os = require('os'); var Profiler = require('./stats/profiler-proxy'); -var StatsD = require('node-statsd').StatsD; var _ = require('underscore'); var LRU = require('lru-cache'); @@ -49,8 +47,8 @@ process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; // override Date.toJSON require('./utils/date_to_json'); -// jshint maxcomplexity:12 -function App() { +// jshint maxcomplexity:9 +function App(statsClient) { var app = express(); @@ -107,45 +105,6 @@ function App() { app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'}))); } - // Initialize statsD client if requested - var statsd_client; - if ( global.settings.statsd ) { - - // Perform keyword substitution in statsd - if ( global.settings.statsd.prefix ) { - var host_token = os.hostname().split('.').reverse().join('.'); - global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, host_token); - } - - statsd_client = new StatsD(global.settings.statsd); - statsd_client.last_error = { msg:'', count:0 }; - statsd_client.socket.on('error', function(err) { - var last_err = statsd_client.last_error; - var last_msg = last_err.msg; - var this_msg = ''+err; - if ( this_msg !== last_msg ) { - console.error("statsd client socket error: " + err); - statsd_client.last_error.count = 1; - statsd_client.last_error.msg = this_msg; - } else { - ++last_err.count; - if ( ! last_err.interval ) { - //console.log("Installing interval"); - statsd_client.last_error.interval = setInterval(function() { - var count = statsd_client.last_error.count; - if ( count > 1 ) { - console.error("last statsd client socket error repeated " + count + " times"); - statsd_client.last_error.count = 1; - //console.log("Clearing interval"); - clearInterval(statsd_client.last_error.interval); - statsd_client.last_error.interval = null; - } - }, 1000); - } - } - }); - } - app.use(cors()); // Use step-profiler @@ -159,7 +118,7 @@ function App() { var profile = global.settings.useProfiler; req.profiler = new Profiler({ profile: profile, - statsd_client: statsd_client + statsd_client: statsClient }); next(); }); @@ -193,10 +152,10 @@ function App() { var genericController = new GenericController(); genericController.route(app); - var queryController = new QueryController(userDatabaseService, tableCache, statsd_client); + var queryController = new QueryController(userDatabaseService, tableCache, statsClient); queryController.route(app); - var jobController = new JobController(userDatabaseService, jobService, statsd_client); + var jobController = new JobController(userDatabaseService, jobService, statsClient); jobController.route(app); var cacheStatusController = new CacheStatusController(tableCache); @@ -213,7 +172,7 @@ function App() { if (global.settings.environment !== 'test' && isBatchProcess) { var batchName = global.settings.api_hostname || 'batch'; app.batch = batchFactory( - metadataBackend, redisPool, batchName, statsd_client, global.settings.batch_log_filename + metadataBackend, redisPool, batchName, statsClient, global.settings.batch_log_filename ); app.batch.start(); } diff --git a/app/stats/client.js b/app/stats/client.js new file mode 100644 index 00000000..614344cb --- /dev/null +++ b/app/stats/client.js @@ -0,0 +1,73 @@ +var _ = require('underscore'); +var debug = require('debug')('windshaft:stats_client'); +var StatsD = require('node-statsd').StatsD; + +module.exports = { + /** + * Returns an StatsD instance or an stub object that replicates the StatsD public interface so there is no need to + * keep checking if the stats_client is instantiated or not. + * + * The first call to this method implies all future calls will use the config specified in the very first call. + * + * TODO: It's far from ideal to use make this a singleton, improvement desired. + * We proceed this way to be able to use StatsD from several places sharing one single StatsD instance. + * + * @param config Configuration for StatsD, if undefined it will return an stub + * @returns {StatsD|Object} + */ + getInstance: function(config) { + + if (!this.instance) { + + var instance; + + if (config) { + instance = new StatsD(config); + instance.last_error = { msg: '', count: 0 }; + instance.socket.on('error', function (err) { + var last_err = instance.last_error; + var last_msg = last_err.msg; + var this_msg = '' + err; + if (this_msg !== last_msg) { + debug("statsd client socket error: " + err); + instance.last_error.count = 1; + instance.last_error.msg = this_msg; + } else { + ++last_err.count; + if (!last_err.interval) { + instance.last_error.interval = setInterval(function () { + var count = instance.last_error.count; + if (count > 1) { + debug("last statsd client socket error repeated " + count + " times"); + instance.last_error.count = 1; + clearInterval(instance.last_error.interval); + instance.last_error.interval = null; + } + }, 1000); + } + } + }); + } else { + var stubFunc = function (stat, value, sampleRate, callback) { + if (_.isFunction(callback)) { + callback(null, 0); + } + }; + instance = { + timing: stubFunc, + increment: stubFunc, + decrement: stubFunc, + gauge: stubFunc, + unique: stubFunc, + set: stubFunc, + sendAll: stubFunc, + send: stubFunc + }; + } + + this.instance = instance; + } + + return this.instance; + } +}; \ No newline at end of file diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index c6bd512b..9db56001 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "cartodb_sql_api", - "version": "1.43.2", + "version": "1.44.1", "dependencies": { "bintrees": { "version": "1.0.1", diff --git a/package.json b/package.json index e0d2924f..c80935bc 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "keywords": [ "cartodb" ], - "version": "1.43.2", + "version": "1.44.1", "repository": { "type": "git", "url": "git://github.com/CartoDB/CartoDB-SQL-API.git"