Split http capacity between simple and load

- Simple will use 'available_cores' from response.
- Load will use 'cores' and 'relative_load'.

batch/scheduler/capacity/http-load.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var util = require('util');
+var debug = require('../../util/debug')('capacity-http-load');
+var HttpSimpleCapacity = require('./http-simple');
+
+function HttpLoadCapacity(host, capacityEndpoint) {
+    HttpSimpleCapacity.call(this);
+    this.host = host;
+    this.capacityEndpoint = capacityEndpoint;
+}
+util.inherits(HttpLoadCapacity, HttpSimpleCapacity);
+
+module.exports = HttpLoadCapacity;
+
+HttpLoadCapacity.prototype.getCapacity = function(callback) {
+    this.getResponse(function(err, values) {
+        var capacity = 1;
+
+        if (err) {
+            return callback(null, capacity);
+        }
+
+        var cores = parseInt(values.cores, 10);
+        var relativeLoad = parseFloat(values.relative_load);
+
+        capacity = Math.max(1, Math.floor(((1 - relativeLoad) * cores) - 1));
+
+        capacity = Number.isFinite(capacity) ? capacity : 1;
+
+        debug('host=%s, capacity=%s', this.host, capacity);
+        return callback(null, capacity);
+    }.bind(this));
+};

batch/scheduler/capacity/http-simple.js

@@ -1,7 +1,7 @@
 'use strict';
 
 var request = require('request');
-var debug = require('../../util/debug')('capacity-http');
+var debug = require('../../util/debug')('capacity-http-simple');
 
 function HttpSimpleCapacity(host, capacityEndpoint) {
     this.host = host;
@@ -11,6 +11,24 @@ function HttpSimpleCapacity(host, capacityEndpoint) {
 module.exports = HttpSimpleCapacity;
 
 HttpSimpleCapacity.prototype.getCapacity = function(callback) {
+    this.getResponse(function(err, values) {
+        var capacity = 1;
+
+        if (err) {
+            return callback(null, capacity);
+        }
+
+        var availableCores = parseInt(values.available_cores, 10);
+
+        capacity = Math.max(availableCores, 1);
+        capacity = Number.isFinite(capacity) ? capacity : 1;
+
+        debug('host=%s, capacity=%s', this.host, capacity);
+        return callback(null, capacity);
+    }.bind(this));
+};
+
+HttpSimpleCapacity.prototype.getResponse = function(callback) {
     var requestParams = {
         method: 'POST',
         url: this.capacityEndpoint,
@@ -18,25 +36,12 @@ HttpSimpleCapacity.prototype.getCapacity = function(callback) {
     };
     debug('getCapacity(%s)', this.host);
     request.post(requestParams, function(err, res, jsonRes) {
-        var capacity = 1;
+        if (err) {
-
+            return callback(err);
-        if (!err && jsonRes) {
-            if (jsonRes.retcode === 0) {
-                var values = jsonRes.return_values;
-
-                var cores = parseInt(values.cores, 10);
-                var relativeLoad = parseFloat(values.relative_load);
-
-                capacity = Math.max(
-                    Math.floor(((1 - relativeLoad) * cores) - 1),
-                    1
-                );
-
-                debug('host=%s, capacity=%s', this.host, capacity);
-            }
         }
-
+        if (jsonRes && jsonRes.retcode === 0) {
-        debug('host=%s, capacity=%s', this.host, capacity);
+            return callback(null, jsonRes.return_values || {});
-        return callback(null, capacity);
+        }
-    }.bind(this));
+        return callback(new Error('Could not retrieve information from endpoint'));
+    });
 };

batch/scheduler/host-scheduler.js

@@ -6,6 +6,7 @@ var Scheduler = require('./scheduler');
 var Locker = require('../leader/locker');
 var FixedCapacity = require('./capacity/fixed');
 var HttpSimpleCapacity = require('./capacity/http-simple');
+var HttpLoadCapacity = require('./capacity/http-load');
 
 function HostScheduler(name, taskRunner, redisPool) {
     this.name = name || 'scheduler';
@@ -35,12 +36,17 @@ HostScheduler.prototype.add = function(host, user, callback) {
 };
 
 HostScheduler.prototype.getCapacityProvider = function(host) {
-    var strategy = global.settings.batch_capacity_strategy || 'fixed';
+    var strategy = global.settings.batch_capacity_strategy;
-    if (strategy === 'http') {
+
+    if (strategy === 'http-simple' || strategy === 'http-load') {
         if (global.settings.batch_capacity_http_url_template) {
             var endpoint = _.template(global.settings.batch_capacity_http_url_template, { dbhost: host });
             debug('Using strategy=%s capacity. Endpoint=%s', strategy, endpoint);
-            return new HttpSimpleCapacity(host, endpoint);
+
+            if (strategy === 'http-simple') {
+                return new HttpSimpleCapacity(host, endpoint);
+            }
+            return new HttpLoadCapacity(host, endpoint);
         }
     }
 

config/environments/development.js.example

@@ -36,15 +36,17 @@ module.exports.batch_log_filename = 'logs/batch-queries.log';
 module.exports.batch_max_queued_jobs = 64;
 // Capacity strategy to use.
 // It allows to tune how many queries run at a db host at the same time.
-// Options: 'fixed', 'http'
+// Options: 'fixed', 'http-simple', 'http-load'
 module.exports.batch_capacity_strategy = 'fixed';
 // Applies when strategy='fixed'.
 // Number of simultaneous users running queries in the same host.
 // It will use 1 as min.
 module.exports.batch_capacity_fixed_amount = 2;
-// Applies when strategy='http'.
+// Applies when strategy='http-simple' or strategy='http-load'.
 // HTTP endpoint to check db host load.
 // Helps to decide the number of simultaneous users running queries in that host.
+// 'http-simple' will use 'available_cores' to decide the number.
+// 'http-load' will use 'cores' and 'relative_load' to decide the number.
 // It will use 1 as min.
 // If no template is provided it will default to 'fixed' strategy.
 module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load';

config/environments/production.js.example

@@ -37,15 +37,17 @@ module.exports.batch_log_filename = 'logs/batch-queries.log';
 module.exports.batch_max_queued_jobs = 64;
 // Capacity strategy to use.
 // It allows to tune how many queries run at a db host at the same time.
-// Options: 'fixed', 'http'
+// Options: 'fixed', 'http-simple', 'http-load'
 module.exports.batch_capacity_strategy = 'fixed';
 // Applies when strategy='fixed'.
 // Number of simultaneous users running queries in the same host.
 // It will use 1 as min.
 module.exports.batch_capacity_fixed_amount = 2;
-// Applies when strategy='http'.
+// Applies when strategy='http-simple' or strategy='http-load'.
 // HTTP endpoint to check db host load.
 // Helps to decide the number of simultaneous users running queries in that host.
+// 'http-simple' will use 'available_cores' to decide the number.
+// 'http-load' will use 'cores' and 'relative_load' to decide the number.
 // It will use 1 as min.
 // If no template is provided it will default to 'fixed' strategy.
 module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load';

config/environments/staging.js.example

@@ -37,15 +37,17 @@ module.exports.batch_log_filename = 'logs/batch-queries.log';
 module.exports.batch_max_queued_jobs = 64;
 // Capacity strategy to use.
 // It allows to tune how many queries run at a db host at the same time.
-// Options: 'fixed', 'http'
+// Options: 'fixed', 'http-simple', 'http-load'
 module.exports.batch_capacity_strategy = 'fixed';
 // Applies when strategy='fixed'.
 // Number of simultaneous users running queries in the same host.
 // It will use 1 as min.
 module.exports.batch_capacity_fixed_amount = 2;
-// Applies when strategy='http'.
+// Applies when strategy='http-simple' or strategy='http-load'.
 // HTTP endpoint to check db host load.
 // Helps to decide the number of simultaneous users running queries in that host.
+// 'http-simple' will use 'available_cores' to decide the number.
+// 'http-load' will use 'cores' and 'relative_load' to decide the number.
 // It will use 1 as min.
 // If no template is provided it will default to 'fixed' strategy.
 module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load';

config/environments/test.js.example

@@ -34,15 +34,17 @@ module.exports.batch_log_filename = 'logs/batch-queries.log';
 module.exports.batch_max_queued_jobs = 64;
 // Capacity strategy to use.
 // It allows to tune how many queries run at a db host at the same time.
-// Options: 'fixed', 'http'
+// Options: 'fixed', 'http-simple', 'http-load'
 module.exports.batch_capacity_strategy = 'fixed';
 // Applies when strategy='fixed'.
 // Number of simultaneous users running queries in the same host.
 // It will use 1 as min.
 module.exports.batch_capacity_fixed_amount = 2;
-// Applies when strategy='http'.
+// Applies when strategy='http-simple' or strategy='http-load'.
 // HTTP endpoint to check db host load.
 // Helps to decide the number of simultaneous users running queries in that host.
+// 'http-simple' will use 'available_cores' to decide the number.
+// 'http-load' will use 'cores' and 'relative_load' to decide the number.
 // It will use 1 as min.
 // If no template is provided it will default to 'fixed' strategy.
 module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load';