Improved queue consuption avoiding possible stack overflow

batch/batch.js

@@ -6,6 +6,7 @@ var JobRunner = require('./job_runner');
 var JobQueuePool = require('./job_queue_pool');
 var JobSubscriber = require('./job_subscriber');
 var UserDatabaseMetadataService = require('./user_database_metadata_service');
+var forever = require('./forever');
 
 function Batch(metadataBackend) {
     EventEmitter.call(this);
@@ -21,42 +22,59 @@ Batch.prototype.start = function () {
     this.jobSubscriber.subscribe(function (channel, host) {
         var queue = self.jobQueuePool.get(host);
 
-        if (!queue) {
-            queue = self.jobQueuePool.add(host);
-            consume(queue);
+        // there is nothing to do. It is already running jobs
+        if (queue) {
+            return;
         }
 
-        function consume(queue) {
+        queue = self.jobQueuePool.add(host);
+
+        // do forever, it does not cause a stack overflow
+        forever(function (next) {
+            self._consume(host, queue, next);
+        }, function (err) {
+            self.jobQueuePool.remove(host);
+
+            if (err.name === 'EmptyQueue') {
+                return console.log(err.message);
+            }
+
+            console.error(err);
+        });
+    });
+};
+
+Batch.prototype._consume = function consume(host, queue, callback) {
+    var self = this;
+
     queue.dequeue(host, function (err, job_id) {
         if (err) {
-                    self.jobQueuePool.remove(host);
-                    return console.error(err);
+            return callback(err);
         }
 
         if (!job_id) {
-                    self.jobQueuePool.remove(host);
-                    return console.log('Queue %s is empty', host);
+            var emptyQueueError = new Error('Queue ' + host + ' is empty');
+            emptyQueueError.name = 'EmptyQueue';
+            return callback(emptyQueueError);
         }
 
         self.jobRunner.run(job_id)
             .on('done', function (job) {
                 console.log('Job %s done in %s', job_id, host);
                 self.emit('job:done', job.job_id);
-                        consume(queue); // recursive call
+                callback();
             })
             .on('failed', function (job) {
-                        console.log('Job %s done in %s', job_id, host);
+                console.log('Job %s failed in %s', job_id, host);
                 self.emit('job:failed', job.job_id);
-                        consume(queue); // recursive call
+                callback();
             })
             .on('error', function (err) {
-                        console.error('Error in job ', err.message || err);
+                console.error('Error in job %s due to', job_id, err.message || err);
                 self.emit('job:failed', job_id);
-                        self.jobQueuePool.remove(host);
+                callback(err);
             });
     });
-        }
-    });
 };
 
 module.exports = Batch;

batch/forever.js

@@ -0,0 +1,11 @@
+'use strict';
+
+module.exports = function forever(fn, done) {
+    function next(err) {
+        if (err) {
+            return done(err);
+        }
+        fn(next);
+    }
+    next();
+};