Default analyses limits can be defined in configuration

This commit is contained in:
Raul Ochoa 2016-10-20 14:02:13 +02:00
parent 30a95b7da3
commit 376573459c
7 changed files with 164 additions and 5 deletions

View File

@ -216,6 +216,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal // Log file will be re-opened on receiving the HUP signal
filename: '/tmp/analysis.log' filename: '/tmp/analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
} }
} }
,millstone: { ,millstone: {

View File

@ -210,6 +210,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal // Log file will be re-opened on receiving the HUP signal
filename: 'logs/analysis.log' filename: 'logs/analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
} }
} }
,millstone: { ,millstone: {

View File

@ -210,6 +210,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal // Log file will be re-opened on receiving the HUP signal
filename: 'logs/analysis.log' filename: 'logs/analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
} }
} }
,millstone: { ,millstone: {

View File

@ -211,6 +211,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal // Log file will be re-opened on receiving the HUP signal
filename: 'node-windshaft.log' filename: 'node-windshaft.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
} }
} }
,millstone: { ,millstone: {

View File

@ -1,3 +1,6 @@
'use strict';
var _ = require('underscore');
var camshaft = require('camshaft'); var camshaft = require('camshaft');
var fs = require('fs'); var fs = require('fs');
@ -8,9 +11,9 @@ var REDIS_LIMITS = {
function AnalysisBackend (metadataBackend, options) { function AnalysisBackend (metadataBackend, options) {
this.metadataBackend = metadataBackend; this.metadataBackend = metadataBackend;
options = options || {}; this.options = options || {};
this.setBatchConfig(options.batch); this.setBatchConfig(this.options.batch);
this.setLoggerConfig(options.logger); this.setLoggerConfig(this.options.logger);
} }
module.exports = AnalysisBackend; module.exports = AnalysisBackend;
@ -55,10 +58,13 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin
}; };
AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) { AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
var self = this;
var analysesLimitsKey = REDIS_LIMITS.PREFIX + username; var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) { this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
analysesTimeouts = analysesTimeouts || {}; analysesTimeouts = analysesTimeouts || {};
_.defaults(analysesTimeouts, self.options.limits);
var analysesLimits = { var analysesLimits = {
analyses: { analyses: {
// buffer: { // buffer: {

View File

@ -39,7 +39,8 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
}, },
logger: { logger: {
filename: undefined filename: undefined
} },
limits: {}
}); });
module.exports = { module.exports = {
@ -101,7 +102,8 @@ module.exports = {
}, },
logger: { logger: {
filename: analysisConfig.logger.filename filename: analysisConfig.logger.filename
} },
limits: analysisConfig.limits
}, },
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161 // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
redis: _.extend(global.environment.redis, {unwatchOnRelease: false}), redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

View File

@ -0,0 +1,127 @@
var testHelper = require('../support/test_helper');
var assert = require('assert');
var redis = require('redis');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
describe('analysis-backend limits', function() {
var redisClient;
var keysToDelete;
var user = 'localhost';
beforeEach(function() {
redisClient = redis.createClient(global.environment.redis.port);
keysToDelete = {};
var redisPool = new RedisPool(global.environment.redis);
this.metadataBackend = cartodbRedis({pool: redisPool});
});
afterEach(function(done) {
redisClient.quit(function() {
testHelper.deleteRedisKeys(keysToDelete, done);
});
});
function withAnalysesLimits(limits, callback) {
redisClient.SELECT(5, function(err) {
if (err) {
return callback(err);
}
var analysesLimitsKey = 'limits:analyses:' + user;
redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
if (err) {
return callback(err);
}
keysToDelete[analysesLimitsKey] = 5;
return callback();
});
});
}
it("should use limits from configuration", function(done) {
var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } });
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
assert.ok(result.analyses.kmeans);
assert.equal(result.analyses.kmeans.timeout, 5000);
done();
});
});
it("should use limits from redis", function(done) {
var self = this;
var limits = ['moran', 5000];
withAnalysesLimits(limits, function(err) {
if (err) {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend);
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
done();
});
});
});
it("should use limits from redis and configuration, redis takes priority", function(done) {
var self = this;
var limits = ['moran', 5000];
withAnalysesLimits(limits, function(err) {
if (err) {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } });
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
done();
});
});
});
it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
var self = this;
var limits = ['moran', 5000];
withAnalysesLimits(limits, function(err) {
if (err) {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } });
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
assert.ok(result.analyses.kmeans);
assert.equal(result.analyses.kmeans.timeout, 1000);
done();
});
});
});
});