Reused matadata backend instance for batch service
This commit is contained in:
parent
fe9280b0f7
commit
d0787d03f7
@ -181,7 +181,7 @@ function App() {
|
|||||||
versionController.route(app);
|
versionController.route(app);
|
||||||
|
|
||||||
if (global.settings.environment !== 'test') {
|
if (global.settings.environment !== 'test') {
|
||||||
batchService(5000, 100);
|
batchService(metadataBackend, 5000, 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
return app;
|
return app;
|
||||||
|
@ -1,21 +1,12 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
var cartoDBRedis = require('cartodb-redis');
|
|
||||||
var UserDatabaseMetadataService = require('./user_database_metadata_service');
|
var UserDatabaseMetadataService = require('./user_database_metadata_service');
|
||||||
var UsernameQueue = require('./username_queue');
|
var UsernameQueue = require('./username_queue');
|
||||||
var JobService = require('./job_service');
|
var JobService = require('./job_service');
|
||||||
var JobCounterService = require('./job_counter_service');
|
var JobCounterService = require('./job_counter_service');
|
||||||
var BatchManager = require('./batch_manager');
|
var BatchManager = require('./batch_manager');
|
||||||
|
|
||||||
module.exports = function (maxJobsPerHost) {
|
module.exports = function (metadataBackend ,maxJobsPerHost) {
|
||||||
var metadataBackend = cartoDBRedis({
|
|
||||||
host: global.settings.redis_host,
|
|
||||||
port: global.settings.redis_port,
|
|
||||||
max: global.settings.redisPool,
|
|
||||||
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
|
|
||||||
reapIntervalMillis: global.settings.redisReapIntervalMillis
|
|
||||||
});
|
|
||||||
|
|
||||||
var usernameQueue = new UsernameQueue(metadataBackend);
|
var usernameQueue = new UsernameQueue(metadataBackend);
|
||||||
var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend);
|
var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend);
|
||||||
var jobService = new JobService(metadataBackend);
|
var jobService = new JobService(metadataBackend);
|
||||||
|
@ -3,8 +3,8 @@
|
|||||||
var BatchLauncher = require('./batch_launcher');
|
var BatchLauncher = require('./batch_launcher');
|
||||||
var batchManagerFactory = require('./batch_manager_factory');
|
var batchManagerFactory = require('./batch_manager_factory');
|
||||||
|
|
||||||
module.exports = function (interval, maxJobsPerHost) {
|
module.exports = function (metadataBackend, interval, maxJobsPerHost) {
|
||||||
var batchManager = batchManagerFactory(maxJobsPerHost);
|
var batchManager = batchManagerFactory(metadataBackend, maxJobsPerHost);
|
||||||
var batchLauncher = new BatchLauncher(batchManager);
|
var batchLauncher = new BatchLauncher(batchManager);
|
||||||
|
|
||||||
// here we go!
|
// here we go!
|
||||||
|
@ -11,26 +11,14 @@ UsernameQueue.prototype.enqueue = function (cdbUsername, callback) {
|
|||||||
var db = this.db;
|
var db = this.db;
|
||||||
var queue = this.queueName;
|
var queue = this.queueName;
|
||||||
|
|
||||||
this.metadataBackend.redisCmd(db, 'LPUSH', [queue, cdbUsername], function (err, cdbUsername) {
|
this.metadataBackend.redisCmd(db, 'LPUSH', [queue, cdbUsername], callback);
|
||||||
if (err) {
|
|
||||||
return callback(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
callback(null, cdbUsername);
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
|
|
||||||
UsernameQueue.prototype.dequeue = function (callback) {
|
UsernameQueue.prototype.dequeue = function (callback) {
|
||||||
var db = this.db;
|
var db = this.db;
|
||||||
var queue = this.queueName;
|
var queue = this.queueName;
|
||||||
|
|
||||||
this.metadataBackend.redisCmd(db, 'RPOP', [queue], function (err, cdbUsername) {
|
this.metadataBackend.redisCmd(db, 'RPOP', [queue], callback);
|
||||||
if (err) {
|
|
||||||
return callback(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
callback(null, cdbUsername);
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = UsernameQueue;
|
module.exports = UsernameQueue;
|
||||||
|
@ -3,7 +3,16 @@ var batchManagerFactory = require('../../batch/batch_manager_factory');
|
|||||||
|
|
||||||
describe('batch manager', function() {
|
describe('batch manager', function() {
|
||||||
it('run', function (done) {
|
it('run', function (done) {
|
||||||
batchManagerFactory().run(function (err) {
|
var metadataBackend = require('cartodb-redis')({
|
||||||
|
host: global.settings.redis_host,
|
||||||
|
port: global.settings.redis_port,
|
||||||
|
max: global.settings.redisPool,
|
||||||
|
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
|
||||||
|
reapIntervalMillis: global.settings.redisReapIntervalMillis
|
||||||
|
});
|
||||||
|
var maxJobsPerHost = 100;
|
||||||
|
|
||||||
|
batchManagerFactory(metadataBackend, maxJobsPerHost).run(function (err) {
|
||||||
done(err);
|
done(err);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
33
test/unit/batch/username_queue.js
Normal file
33
test/unit/batch/username_queue.js
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
var UsernameQueue = require('../../../batch/username_queue');
|
||||||
|
var assert = require('assert');
|
||||||
|
|
||||||
|
describe('batch API username queue', function () {
|
||||||
|
beforeEach(function () {
|
||||||
|
this.metadataBackend = {
|
||||||
|
redisCmd: function () {
|
||||||
|
var callback = arguments[arguments.length -1];
|
||||||
|
process.nextTick(function () {
|
||||||
|
callback(null, 'irrelevantUsername');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
this.usernameQueue = new UsernameQueue(this.metadataBackend);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('.enqueue(username) should enqueue the provided username', function (done) {
|
||||||
|
this.usernameQueue.enqueue('irrelevantUsername', function (err, username) {
|
||||||
|
assert.ok(!err);
|
||||||
|
assert.equal(username, 'irrelevantUsername');
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('.dequeue(username) should dequeue the next username', function (done) {
|
||||||
|
this.usernameQueue.dequeue(function (err, username) {
|
||||||
|
assert.ok(!err);
|
||||||
|
assert.equal(username, 'irrelevantUsername');
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
Loading…
Reference in New Issue
Block a user