Merge branch 'master' into avoid-scan-to-discover-queues

NEWS.md

@@ -1,9 +1,16 @@
 #Changelog
 
-## 1.43.2
+## 1.44.1
 Released 2017-mm-dd
 
 
+## 1.44.0
+Released 2017-03-30
+
+Announcements:
+ * Active GC interval for Node.js >=v6.
+
+
 ## 1.43.1
 Released 2017-01-16
 

app.js

@@ -11,6 +11,7 @@
 */
 var fs = require('fs');
 var path = require('path');
+var os = require('os');
 
 var argv = require('yargs')
     .usage('Usage: $0 <environment> [options]')
@@ -80,7 +81,17 @@ if ( ! global.settings.base_url ) {
 
 var version = require("./package").version;
 
-var server = require('./app/server')();
+var StatsClient = require('./app/stats/client');
+if (global.settings.statsd) {
+    // Perform keyword substitution in statsd
+    if (global.settings.statsd.prefix) {
+        var hostToken = os.hostname().split('.').reverse().join('.');
+        global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, hostToken);
+    }
+}
+var statsClient = StatsClient.getInstance(global.settings.statsd);
+
+var server = require('./app/server')(statsClient);
 var listener = server.listen(global.settings.node_port, global.settings.node_host);
 listener.on('listening', function() {
     console.info("Using Node.js %s", process.version);
@@ -119,3 +130,28 @@ process.on('SIGTERM', function () {
         process.exit(0);
     });
 });
+
+function isGteMinVersion(version, minVersion) {
+    var versionMatch = /[a-z]?([0-9]*)/.exec(version);
+    if (versionMatch) {
+        var majorVersion = parseInt(versionMatch[1], 10);
+        if (Number.isFinite(majorVersion)) {
+            return majorVersion >= minVersion;
+        }
+    }
+    return false;
+}
+
+if (global.gc && isGteMinVersion(process.version, 6)) {
+    var gcInterval = Number.isFinite(global.settings.gc_interval) ?
+        global.settings.gc_interval :
+        10000;
+
+    if (gcInterval > 0) {
+        setInterval(function gcForcedCycle() {
+            var start = Date.now();
+            global.gc();
+            statsClient.timing('sqlapi.gc', Date.now() - start);
+        }, gcInterval);
+    }
+}

app/server.js

@@ -16,9 +16,7 @@
 
 var express = require('express');
 var bodyParser = require('./middlewares/body-parser');
-var os = require('os');
 var Profiler = require('./stats/profiler-proxy');
-var StatsD = require('node-statsd').StatsD;
 var _ = require('underscore');
 var LRU = require('lru-cache');
 
@@ -49,8 +47,8 @@ process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi';
 // override Date.toJSON
 require('./utils/date_to_json');
 
-// jshint maxcomplexity:12
+// jshint maxcomplexity:9
-function App() {
+function App(statsClient) {
 
     var app = express();
 
@@ -107,45 +105,6 @@ function App() {
         app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'})));
     }
 
-    // Initialize statsD client if requested
-    var statsd_client;
-    if ( global.settings.statsd ) {
-
-      // Perform keyword substitution in statsd
-      if ( global.settings.statsd.prefix ) {
-        var host_token = os.hostname().split('.').reverse().join('.');
-        global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, host_token);
-      }
-
-      statsd_client = new StatsD(global.settings.statsd);
-      statsd_client.last_error = { msg:'', count:0 };
-      statsd_client.socket.on('error', function(err) {
-        var last_err = statsd_client.last_error;
-        var last_msg = last_err.msg;
-        var this_msg = ''+err;
-        if ( this_msg !== last_msg ) {
-          console.error("statsd client socket error: " + err);
-          statsd_client.last_error.count = 1;
-          statsd_client.last_error.msg = this_msg;
-        } else {
-            ++last_err.count;
-            if ( ! last_err.interval ) {
-              //console.log("Installing interval");
-              statsd_client.last_error.interval = setInterval(function() {
-                var count = statsd_client.last_error.count;
-                if ( count > 1 ) {
-                  console.error("last statsd client socket error repeated " + count + " times");
-                  statsd_client.last_error.count = 1;
-                  //console.log("Clearing interval");
-                  clearInterval(statsd_client.last_error.interval);
-                  statsd_client.last_error.interval = null;
-                }
-              }, 1000);
-            }
-        }
-      });
-    }
-
     app.use(cors());
 
     // Use step-profiler
@@ -159,7 +118,7 @@ function App() {
         var profile = global.settings.useProfiler;
         req.profiler = new Profiler({
             profile: profile,
-            statsd_client: statsd_client
+            statsd_client: statsClient
         });
         next();
     });
@@ -193,10 +152,10 @@ function App() {
     var genericController = new GenericController();
     genericController.route(app);
 
-    var queryController = new QueryController(userDatabaseService, tableCache, statsd_client);
+    var queryController = new QueryController(userDatabaseService, tableCache, statsClient);
     queryController.route(app);
 
-    var jobController = new JobController(userDatabaseService, jobService, statsd_client);
+    var jobController = new JobController(userDatabaseService, jobService, statsClient);
     jobController.route(app);
 
     var cacheStatusController = new CacheStatusController(tableCache);
@@ -213,7 +172,7 @@ function App() {
     if (global.settings.environment !== 'test' && isBatchProcess) {
         var batchName = global.settings.api_hostname || 'batch';
         app.batch = batchFactory(
-            metadataBackend, redisPool, batchName, statsd_client, global.settings.batch_log_filename
+            metadataBackend, redisPool, batchName, statsClient, global.settings.batch_log_filename
         );
         app.batch.start();
     }

app/stats/client.js

@@ -0,0 +1,73 @@
+var _ = require('underscore');
+var debug = require('debug')('windshaft:stats_client');
+var StatsD = require('node-statsd').StatsD;
+
+module.exports = {
+    /**
+     * Returns an StatsD instance or an stub object that replicates the StatsD public interface so there is no need to
+     * keep checking if the stats_client is instantiated or not.
+     *
+     * The first call to this method implies all future calls will use the config specified in the very first call.
+     *
+     * TODO: It's far from ideal to use make this a singleton, improvement desired.
+     * We proceed this way to be able to use StatsD from several places sharing one single StatsD instance.
+     *
+     * @param config Configuration for StatsD, if undefined it will return an stub
+     * @returns {StatsD|Object}
+     */
+    getInstance: function(config) {
+
+        if (!this.instance) {
+
+            var instance;
+
+            if (config) {
+                instance = new StatsD(config);
+                instance.last_error = { msg: '', count: 0 };
+                instance.socket.on('error', function (err) {
+                    var last_err = instance.last_error;
+                    var last_msg = last_err.msg;
+                    var this_msg = '' + err;
+                    if (this_msg !== last_msg) {
+                        debug("statsd client socket error: " + err);
+                        instance.last_error.count = 1;
+                        instance.last_error.msg = this_msg;
+                    } else {
+                        ++last_err.count;
+                        if (!last_err.interval) {
+                            instance.last_error.interval = setInterval(function () {
+                                var count = instance.last_error.count;
+                                if (count > 1) {
+                                    debug("last statsd client socket error repeated " + count + " times");
+                                    instance.last_error.count = 1;
+                                    clearInterval(instance.last_error.interval);
+                                    instance.last_error.interval = null;
+                                }
+                            }, 1000);
+                        }
+                    }
+                });
+            } else {
+                var stubFunc = function (stat, value, sampleRate, callback) {
+                    if (_.isFunction(callback)) {
+                        callback(null, 0);
+                    }
+                };
+                instance = {
+                    timing: stubFunc,
+                    increment: stubFunc,
+                    decrement: stubFunc,
+                    gauge: stubFunc,
+                    unique: stubFunc,
+                    set: stubFunc,
+                    sendAll: stubFunc,
+                    send: stubFunc
+                };
+            }
+
+            this.instance = instance;
+        }
+
+        return this.instance;
+    }
+};

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "cartodb_sql_api",
-  "version": "1.43.2",
+  "version": "1.44.1",
   "dependencies": {
     "bintrees": {
       "version": "1.0.1",

package.json

@@ -5,7 +5,7 @@
   "keywords": [
     "cartodb"
   ],
-  "version": "1.43.2",
+  "version": "1.44.1",
   "repository": {
     "type": "git",
     "url": "git://github.com/CartoDB/CartoDB-SQL-API.git"