Default analyses limits can be defined in configuration
This commit is contained in:
parent
30a95b7da3
commit
376573459c
@ -216,6 +216,12 @@ var config = {
|
|||||||
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
||||||
// Log file will be re-opened on receiving the HUP signal
|
// Log file will be re-opened on receiving the HUP signal
|
||||||
filename: '/tmp/analysis.log'
|
filename: '/tmp/analysis.log'
|
||||||
|
},
|
||||||
|
// Define max execution time in ms for analyses or tags
|
||||||
|
// If analysis or tag are not found in redis this values will be used as default.
|
||||||
|
limits: {
|
||||||
|
moran: 120000,
|
||||||
|
cpu2x: 60000
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
,millstone: {
|
,millstone: {
|
||||||
|
@ -210,6 +210,12 @@ var config = {
|
|||||||
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
||||||
// Log file will be re-opened on receiving the HUP signal
|
// Log file will be re-opened on receiving the HUP signal
|
||||||
filename: 'logs/analysis.log'
|
filename: 'logs/analysis.log'
|
||||||
|
},
|
||||||
|
// Define max execution time in ms for analyses or tags
|
||||||
|
// If analysis or tag are not found in redis this values will be used as default.
|
||||||
|
limits: {
|
||||||
|
moran: 120000,
|
||||||
|
cpu2x: 60000
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
,millstone: {
|
,millstone: {
|
||||||
|
@ -210,6 +210,12 @@ var config = {
|
|||||||
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
||||||
// Log file will be re-opened on receiving the HUP signal
|
// Log file will be re-opened on receiving the HUP signal
|
||||||
filename: 'logs/analysis.log'
|
filename: 'logs/analysis.log'
|
||||||
|
},
|
||||||
|
// Define max execution time in ms for analyses or tags
|
||||||
|
// If analysis or tag are not found in redis this values will be used as default.
|
||||||
|
limits: {
|
||||||
|
moran: 120000,
|
||||||
|
cpu2x: 60000
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
,millstone: {
|
,millstone: {
|
||||||
|
@ -211,6 +211,12 @@ var config = {
|
|||||||
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
|
||||||
// Log file will be re-opened on receiving the HUP signal
|
// Log file will be re-opened on receiving the HUP signal
|
||||||
filename: 'node-windshaft.log'
|
filename: 'node-windshaft.log'
|
||||||
|
},
|
||||||
|
// Define max execution time in ms for analyses or tags
|
||||||
|
// If analysis or tag are not found in redis this values will be used as default.
|
||||||
|
limits: {
|
||||||
|
moran: 120000,
|
||||||
|
cpu2x: 60000
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
,millstone: {
|
,millstone: {
|
||||||
|
@ -1,3 +1,6 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
var _ = require('underscore');
|
||||||
var camshaft = require('camshaft');
|
var camshaft = require('camshaft');
|
||||||
var fs = require('fs');
|
var fs = require('fs');
|
||||||
|
|
||||||
@ -8,9 +11,9 @@ var REDIS_LIMITS = {
|
|||||||
|
|
||||||
function AnalysisBackend (metadataBackend, options) {
|
function AnalysisBackend (metadataBackend, options) {
|
||||||
this.metadataBackend = metadataBackend;
|
this.metadataBackend = metadataBackend;
|
||||||
options = options || {};
|
this.options = options || {};
|
||||||
this.setBatchConfig(options.batch);
|
this.setBatchConfig(this.options.batch);
|
||||||
this.setLoggerConfig(options.logger);
|
this.setLoggerConfig(this.options.logger);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = AnalysisBackend;
|
module.exports = AnalysisBackend;
|
||||||
@ -55,10 +58,13 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin
|
|||||||
};
|
};
|
||||||
|
|
||||||
AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
|
AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
|
||||||
|
var self = this;
|
||||||
var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
|
var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
|
||||||
this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
|
this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
|
||||||
analysesTimeouts = analysesTimeouts || {};
|
analysesTimeouts = analysesTimeouts || {};
|
||||||
|
|
||||||
|
_.defaults(analysesTimeouts, self.options.limits);
|
||||||
|
|
||||||
var analysesLimits = {
|
var analysesLimits = {
|
||||||
analyses: {
|
analyses: {
|
||||||
// buffer: {
|
// buffer: {
|
||||||
|
@ -39,7 +39,8 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
|
|||||||
},
|
},
|
||||||
logger: {
|
logger: {
|
||||||
filename: undefined
|
filename: undefined
|
||||||
}
|
},
|
||||||
|
limits: {}
|
||||||
});
|
});
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
@ -101,7 +102,8 @@ module.exports = {
|
|||||||
},
|
},
|
||||||
logger: {
|
logger: {
|
||||||
filename: analysisConfig.logger.filename
|
filename: analysisConfig.logger.filename
|
||||||
}
|
},
|
||||||
|
limits: analysisConfig.limits
|
||||||
},
|
},
|
||||||
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
|
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
|
||||||
redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),
|
redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),
|
||||||
|
127
test/integration/analysis-backend-limits.js
Normal file
127
test/integration/analysis-backend-limits.js
Normal file
@ -0,0 +1,127 @@
|
|||||||
|
var testHelper = require('../support/test_helper');
|
||||||
|
|
||||||
|
var assert = require('assert');
|
||||||
|
var redis = require('redis');
|
||||||
|
|
||||||
|
var RedisPool = require('redis-mpool');
|
||||||
|
var cartodbRedis = require('cartodb-redis');
|
||||||
|
|
||||||
|
var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
|
||||||
|
|
||||||
|
describe('analysis-backend limits', function() {
|
||||||
|
|
||||||
|
var redisClient;
|
||||||
|
var keysToDelete;
|
||||||
|
var user = 'localhost';
|
||||||
|
|
||||||
|
beforeEach(function() {
|
||||||
|
redisClient = redis.createClient(global.environment.redis.port);
|
||||||
|
keysToDelete = {};
|
||||||
|
var redisPool = new RedisPool(global.environment.redis);
|
||||||
|
this.metadataBackend = cartodbRedis({pool: redisPool});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(function(done) {
|
||||||
|
redisClient.quit(function() {
|
||||||
|
testHelper.deleteRedisKeys(keysToDelete, done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
function withAnalysesLimits(limits, callback) {
|
||||||
|
redisClient.SELECT(5, function(err) {
|
||||||
|
if (err) {
|
||||||
|
return callback(err);
|
||||||
|
}
|
||||||
|
var analysesLimitsKey = 'limits:analyses:' + user;
|
||||||
|
redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
|
||||||
|
if (err) {
|
||||||
|
return callback(err);
|
||||||
|
}
|
||||||
|
keysToDelete[analysesLimitsKey] = 5;
|
||||||
|
return callback();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
it("should use limits from configuration", function(done) {
|
||||||
|
var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } });
|
||||||
|
analysisBackend.getAnalysesLimits(user, function(err, result) {
|
||||||
|
assert.ok(!err, err);
|
||||||
|
|
||||||
|
assert.ok(result.analyses.moran);
|
||||||
|
assert.equal(result.analyses.moran.timeout, 5000);
|
||||||
|
|
||||||
|
assert.ok(result.analyses.kmeans);
|
||||||
|
assert.equal(result.analyses.kmeans.timeout, 5000);
|
||||||
|
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should use limits from redis", function(done) {
|
||||||
|
var self = this;
|
||||||
|
var limits = ['moran', 5000];
|
||||||
|
|
||||||
|
withAnalysesLimits(limits, function(err) {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
var analysisBackend = new AnalysisBackend(self.metadataBackend);
|
||||||
|
analysisBackend.getAnalysesLimits(user, function(err, result) {
|
||||||
|
assert.ok(!err, err);
|
||||||
|
|
||||||
|
assert.ok(result.analyses.moran);
|
||||||
|
assert.equal(result.analyses.moran.timeout, 5000);
|
||||||
|
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should use limits from redis and configuration, redis takes priority", function(done) {
|
||||||
|
var self = this;
|
||||||
|
var limits = ['moran', 5000];
|
||||||
|
|
||||||
|
withAnalysesLimits(limits, function(err) {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } });
|
||||||
|
analysisBackend.getAnalysesLimits(user, function(err, result) {
|
||||||
|
assert.ok(!err, err);
|
||||||
|
|
||||||
|
assert.ok(result.analyses.moran);
|
||||||
|
assert.equal(result.analyses.moran.timeout, 5000);
|
||||||
|
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
|
||||||
|
var self = this;
|
||||||
|
var limits = ['moran', 5000];
|
||||||
|
|
||||||
|
withAnalysesLimits(limits, function(err) {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } });
|
||||||
|
analysisBackend.getAnalysesLimits(user, function(err, result) {
|
||||||
|
assert.ok(!err, err);
|
||||||
|
|
||||||
|
assert.ok(result.analyses.moran);
|
||||||
|
assert.equal(result.analyses.moran.timeout, 5000);
|
||||||
|
|
||||||
|
assert.ok(result.analyses.kmeans);
|
||||||
|
assert.equal(result.analyses.kmeans.timeout, 1000);
|
||||||
|
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
Loading…
Reference in New Issue
Block a user