cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Default analyses limits can be defined in configuration

Browse Source
This commit is contained in:
Raul Ochoa 2016-10-20 14:02:13 +02:00
parent 30a95b7da3
commit 376573459c
7 changed files with 164 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/environments/development.js.example
View File

@ -216,6 +216,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: '/tmp/analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
}
}
,millstone: {

6
config/environments/production.js.example
View File

@ -210,6 +210,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'logs/analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
}
}
,millstone: {

6
config/environments/staging.js.example
View File

@ -210,6 +210,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'logs/analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
}
}
,millstone: {

6
config/environments/test.js.example
View File

@ -211,6 +211,12 @@ var config = {
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'node-windshaft.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
}
}
,millstone: {

12
lib/cartodb/backends/analysis.js
View File

@ -1,3 +1,6 @@
'use strict';
var _ = require('underscore');
var camshaft = require('camshaft');
var fs = require('fs');
@ -8,9 +11,9 @@ var REDIS_LIMITS = {
function AnalysisBackend (metadataBackend, options) {
this.metadataBackend = metadataBackend;
options = options || {};
this.setBatchConfig(options.batch);
this.setLoggerConfig(options.logger);
this.options = options || {};
this.setBatchConfig(this.options.batch);
this.setLoggerConfig(this.options.logger);
}
module.exports = AnalysisBackend;
@ -55,10 +58,13 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin
};
AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
var self = this;
var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
analysesTimeouts = analysesTimeouts || {};
_.defaults(analysesTimeouts, self.options.limits);
var analysesLimits = {
analyses: {
// buffer: {

6
lib/cartodb/server_options.js
View File

@ -39,7 +39,8 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
},
logger: {
filename: undefined
}
},
limits: {}
});
module.exports = {
@ -101,7 +102,8 @@ module.exports = {
},
logger: {
filename: analysisConfig.logger.filename
}
},
limits: analysisConfig.limits
},
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

127
test/integration/analysis-backend-limits.js Normal file
View File

@ -0,0 +1,127 @@
var testHelper = require('../support/test_helper');
var assert = require('assert');
var redis = require('redis');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
describe('analysis-backend limits', function() {
var redisClient;
var keysToDelete;
var user = 'localhost';
beforeEach(function() {
redisClient = redis.createClient(global.environment.redis.port);
keysToDelete = {};
var redisPool = new RedisPool(global.environment.redis);
this.metadataBackend = cartodbRedis({pool: redisPool});
});
afterEach(function(done) {
redisClient.quit(function() {
testHelper.deleteRedisKeys(keysToDelete, done);
});
});
function withAnalysesLimits(limits, callback) {
redisClient.SELECT(5, function(err) {
if (err) {
return callback(err);
}
var analysesLimitsKey = 'limits:analyses:' + user;
redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
if (err) {
return callback(err);
}
keysToDelete[analysesLimitsKey] = 5;
return callback();
});
});
}
it("should use limits from configuration", function(done) {
var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } });
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
assert.ok(result.analyses.kmeans);
assert.equal(result.analyses.kmeans.timeout, 5000);
done();
});
});
it("should use limits from redis", function(done) {
var self = this;
var limits = ['moran', 5000];
withAnalysesLimits(limits, function(err) {
if (err) {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend);
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
done();
});
});
});
it("should use limits from redis and configuration, redis takes priority", function(done) {
var self = this;
var limits = ['moran', 5000];
withAnalysesLimits(limits, function(err) {
if (err) {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } });
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
done();
});
});
});
it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
var self = this;
var limits = ['moran', 5000];
withAnalysesLimits(limits, function(err) {
if (err) {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } });
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses.moran);
assert.equal(result.analyses.moran.timeout, 5000);
assert.ok(result.analyses.kmeans);
assert.equal(result.analyses.kmeans.timeout, 1000);
done();
});
});
});
});