cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Analyses limit configuration allows to set other limits than timeout

Browse Source 
Configuration is now defined as a dictionary instead of just timeouts
per analysis type
This commit is contained in:
Raul Ochoa 2016-11-10 18:41:59 +01:00
parent 6901b2049e
commit e74ce9dfd8
6 changed files with 73 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/environments/development.js.example
View File

@ -220,8 +220,8 @@ var config = {
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
moran: { timeout: 120000, maxNumberOfRows: 1e5 },
cpu2x: { timeout: 60000 }
}
}
,millstone: {

4
config/environments/production.js.example
View File

@ -214,8 +214,8 @@ var config = {
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
moran: { timeout: 120000, maxNumberOfRows: 1e5 },
cpu2x: { timeout: 60000 }
}
}
,millstone: {

4
config/environments/staging.js.example
View File

@ -214,8 +214,8 @@ var config = {
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
moran: { timeout: 120000, maxNumberOfRows: 1e5 },
cpu2x: { timeout: 60000 }
}
}
,millstone: {

4
config/environments/test.js.example
View File

@ -215,8 +215,8 @@ var config = {
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: 120000,
cpu2x: 60000
moran: { timeout: 120000, maxNumberOfRows: 1e5 },
cpu2x: { timeout: 60000 }
}
}
,millstone: {

35
lib/cartodb/backends/analysis.js
View File

@ -12,6 +12,7 @@ var REDIS_LIMITS = {
function AnalysisBackend (metadataBackend, options) {
this.metadataBackend = metadataBackend;
this.options = options || {};
this.options.limits = this.options.limits || {};
this.setBatchConfig(this.options.batch);
this.setLoggerConfig(this.options.logger);
}
@ -59,24 +60,32 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin
AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
var self = this;
var analysesLimits = {
analyses: {
// buffer: {
// timeout: 1000,
// maxNumberOfRows: 1e6
// }
}
};
Object.keys(self.options.limits).forEach(function(analysisTypeOrTag) {
analysesLimits.analyses[analysisTypeOrTag] = _.extend({}, self.options.limits[analysisTypeOrTag]);
});
var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
// analysesTimeouts wil be something like: { moran: 3000, intersection: 5000 }
analysesTimeouts = analysesTimeouts || {};
_.defaults(analysesTimeouts, self.options.limits);
var analysesLimits = {
analyses: {
// buffer: {
// timeout: 1000
// }
}
};
Object.keys(analysesTimeouts).forEach(function(analysisType) {
analysesLimits.analyses[analysisType] = {
timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
};
analysesLimits.analyses[analysisType] = _.defaults(
{
timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
},
analysesLimits.analyses[analysisType]
);
});
return callback(null, analysesLimits);

46
test/integration/analysis-backend-limits.js
View File

@ -44,7 +44,12 @@ describe('analysis-backend limits', function() {
}
it("should use limits from configuration", function(done) {
var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } });
var analysisBackend = new AnalysisBackend(this.metadataBackend, {
limits: {
moran: { timeout: 5000 },
kmeans: { timeout: 5000 }
}
});
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
@ -88,7 +93,11 @@ describe('analysis-backend limits', function() {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } });
var analysisBackend = new AnalysisBackend(self.metadataBackend, {
limits: {
moran: { timeout: 1000 }
}
});
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
@ -109,7 +118,12 @@ describe('analysis-backend limits', function() {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } });
var analysisBackend = new AnalysisBackend(self.metadataBackend, {
limits: {
moran: { timeout: 1000 },
kmeans: { timeout: 1000 }
}
});
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
@ -124,4 +138,30 @@ describe('analysis-backend limits', function() {
});
});
it("should allow to set other limits per analysis via configuration, and keep timeout from redis", function(done) {
var self = this;
var limits = ['aggregate-intersection', 5000];
withAnalysesLimits(limits, function(err) {
if (err) {
return done(err);
}
var analysisBackend = new AnalysisBackend(self.metadataBackend, {
limits: {
'aggregate-intersection': { timeout: 10000, maxNumberOfRows: 1e5 }
}
});
analysisBackend.getAnalysesLimits(user, function(err, result) {
assert.ok(!err, err);
assert.ok(result.analyses['aggregate-intersection']);
assert.equal(result.analyses['aggregate-intersection'].timeout, 5000);
assert.equal(result.analyses['aggregate-intersection'].maxNumberOfRows, 1e5);
done();
});
});
});
});