From e74ce9dfd8c6b49debf3fa0042c14ba985c3dc3f Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 10 Nov 2016 18:41:59 +0100 Subject: [PATCH] Analyses limit configuration allows to set other limits than timeout Configuration is now defined as a dictionary instead of just timeouts per analysis type --- config/environments/development.js.example | 4 +- config/environments/production.js.example | 4 +- config/environments/staging.js.example | 4 +- config/environments/test.js.example | 4 +- lib/cartodb/backends/analysis.js | 35 ++++++++++------ test/integration/analysis-backend-limits.js | 46 +++++++++++++++++++-- 6 files changed, 73 insertions(+), 24 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 03ab9185..be5b2350 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -220,8 +220,8 @@ var config = { // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. limits: { - moran: 120000, - cpu2x: 60000 + moran: { timeout: 120000, maxNumberOfRows: 1e5 }, + cpu2x: { timeout: 60000 } } } ,millstone: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 1c6feebc..bcf2f8f2 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -214,8 +214,8 @@ var config = { // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. limits: { - moran: 120000, - cpu2x: 60000 + moran: { timeout: 120000, maxNumberOfRows: 1e5 }, + cpu2x: { timeout: 60000 } } } ,millstone: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 2743ec7b..e3681a68 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -214,8 +214,8 @@ var config = { // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. limits: { - moran: 120000, - cpu2x: 60000 + moran: { timeout: 120000, maxNumberOfRows: 1e5 }, + cpu2x: { timeout: 60000 } } } ,millstone: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 975b1b6c..49e59217 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -215,8 +215,8 @@ var config = { // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. limits: { - moran: 120000, - cpu2x: 60000 + moran: { timeout: 120000, maxNumberOfRows: 1e5 }, + cpu2x: { timeout: 60000 } } } ,millstone: { diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js index 086be5da..4abaf4c7 100644 --- a/lib/cartodb/backends/analysis.js +++ b/lib/cartodb/backends/analysis.js @@ -12,6 +12,7 @@ var REDIS_LIMITS = { function AnalysisBackend (metadataBackend, options) { this.metadataBackend = metadataBackend; this.options = options || {}; + this.options.limits = this.options.limits || {}; this.setBatchConfig(this.options.batch); this.setLoggerConfig(this.options.logger); } @@ -59,24 +60,32 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) { var self = this; + + var analysesLimits = { + analyses: { + // buffer: { + // timeout: 1000, + // maxNumberOfRows: 1e6 + // } + } + }; + + Object.keys(self.options.limits).forEach(function(analysisTypeOrTag) { + analysesLimits.analyses[analysisTypeOrTag] = _.extend({}, self.options.limits[analysisTypeOrTag]); + }); + var analysesLimitsKey = REDIS_LIMITS.PREFIX + username; this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) { + // analysesTimeouts wil be something like: { moran: 3000, intersection: 5000 } analysesTimeouts = analysesTimeouts || {}; - _.defaults(analysesTimeouts, self.options.limits); - - var analysesLimits = { - analyses: { - // buffer: { - // timeout: 1000 - // } - } - }; - Object.keys(analysesTimeouts).forEach(function(analysisType) { - analysesLimits.analyses[analysisType] = { - timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0 - }; + analysesLimits.analyses[analysisType] = _.defaults( + { + timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0 + }, + analysesLimits.analyses[analysisType] + ); }); return callback(null, analysesLimits); diff --git a/test/integration/analysis-backend-limits.js b/test/integration/analysis-backend-limits.js index 7308ac30..720bdd30 100644 --- a/test/integration/analysis-backend-limits.js +++ b/test/integration/analysis-backend-limits.js @@ -44,7 +44,12 @@ describe('analysis-backend limits', function() { } it("should use limits from configuration", function(done) { - var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } }); + var analysisBackend = new AnalysisBackend(this.metadataBackend, { + limits: { + moran: { timeout: 5000 }, + kmeans: { timeout: 5000 } + } + }); analysisBackend.getAnalysesLimits(user, function(err, result) { assert.ok(!err, err); @@ -88,7 +93,11 @@ describe('analysis-backend limits', function() { return done(err); } - var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } }); + var analysisBackend = new AnalysisBackend(self.metadataBackend, { + limits: { + moran: { timeout: 1000 } + } + }); analysisBackend.getAnalysesLimits(user, function(err, result) { assert.ok(!err, err); @@ -109,7 +118,12 @@ describe('analysis-backend limits', function() { return done(err); } - var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } }); + var analysisBackend = new AnalysisBackend(self.metadataBackend, { + limits: { + moran: { timeout: 1000 }, + kmeans: { timeout: 1000 } + } + }); analysisBackend.getAnalysesLimits(user, function(err, result) { assert.ok(!err, err); @@ -124,4 +138,30 @@ describe('analysis-backend limits', function() { }); }); + it("should allow to set other limits per analysis via configuration, and keep timeout from redis", function(done) { + var self = this; + var limits = ['aggregate-intersection', 5000]; + + withAnalysesLimits(limits, function(err) { + if (err) { + return done(err); + } + + var analysisBackend = new AnalysisBackend(self.metadataBackend, { + limits: { + 'aggregate-intersection': { timeout: 10000, maxNumberOfRows: 1e5 } + } + }); + analysisBackend.getAnalysesLimits(user, function(err, result) { + assert.ok(!err, err); + + assert.ok(result.analyses['aggregate-intersection']); + assert.equal(result.analyses['aggregate-intersection'].timeout, 5000); + assert.equal(result.analyses['aggregate-intersection'].maxNumberOfRows, 1e5); + + done(); + }); + }); + }); + });