From 376573459cfecdfd33f47c33fc4ecfcefbe356b3 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 20 Oct 2016 14:02:13 +0200 Subject: [PATCH] Default analyses limits can be defined in configuration --- config/environments/development.js.example | 6 + config/environments/production.js.example | 6 + config/environments/staging.js.example | 6 + config/environments/test.js.example | 6 + lib/cartodb/backends/analysis.js | 12 +- lib/cartodb/server_options.js | 6 +- test/integration/analysis-backend-limits.js | 127 ++++++++++++++++++++ 7 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 test/integration/analysis-backend-limits.js diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 1432b31f..03ab9185 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -216,6 +216,12 @@ var config = { // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal filename: '/tmp/analysis.log' + }, + // Define max execution time in ms for analyses or tags + // If analysis or tag are not found in redis this values will be used as default. + limits: { + moran: 120000, + cpu2x: 60000 } } ,millstone: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index e7f8eb21..1c6feebc 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -210,6 +210,12 @@ var config = { // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal filename: 'logs/analysis.log' + }, + // Define max execution time in ms for analyses or tags + // If analysis or tag are not found in redis this values will be used as default. + limits: { + moran: 120000, + cpu2x: 60000 } } ,millstone: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 08fd5c98..2743ec7b 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -210,6 +210,12 @@ var config = { // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal filename: 'logs/analysis.log' + }, + // Define max execution time in ms for analyses or tags + // If analysis or tag are not found in redis this values will be used as default. + limits: { + moran: 120000, + cpu2x: 60000 } } ,millstone: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 1350f7e6..975b1b6c 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -211,6 +211,12 @@ var config = { // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal filename: 'node-windshaft.log' + }, + // Define max execution time in ms for analyses or tags + // If analysis or tag are not found in redis this values will be used as default. + limits: { + moran: 120000, + cpu2x: 60000 } } ,millstone: { diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js index 2194dd75..086be5da 100644 --- a/lib/cartodb/backends/analysis.js +++ b/lib/cartodb/backends/analysis.js @@ -1,3 +1,6 @@ +'use strict'; + +var _ = require('underscore'); var camshaft = require('camshaft'); var fs = require('fs'); @@ -8,9 +11,9 @@ var REDIS_LIMITS = { function AnalysisBackend (metadataBackend, options) { this.metadataBackend = metadataBackend; - options = options || {}; - this.setBatchConfig(options.batch); - this.setLoggerConfig(options.logger); + this.options = options || {}; + this.setBatchConfig(this.options.batch); + this.setLoggerConfig(this.options.logger); } module.exports = AnalysisBackend; @@ -55,10 +58,13 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin }; AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) { + var self = this; var analysesLimitsKey = REDIS_LIMITS.PREFIX + username; this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) { analysesTimeouts = analysesTimeouts || {}; + _.defaults(analysesTimeouts, self.options.limits); + var analysesLimits = { analyses: { // buffer: { diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 052e30cc..c5d5d41b 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -39,7 +39,8 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, { }, logger: { filename: undefined - } + }, + limits: {} }); module.exports = { @@ -101,7 +102,8 @@ module.exports = { }, logger: { filename: analysisConfig.logger.filename - } + }, + limits: analysisConfig.limits }, // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161 redis: _.extend(global.environment.redis, {unwatchOnRelease: false}), diff --git a/test/integration/analysis-backend-limits.js b/test/integration/analysis-backend-limits.js new file mode 100644 index 00000000..7308ac30 --- /dev/null +++ b/test/integration/analysis-backend-limits.js @@ -0,0 +1,127 @@ +var testHelper = require('../support/test_helper'); + +var assert = require('assert'); +var redis = require('redis'); + +var RedisPool = require('redis-mpool'); +var cartodbRedis = require('cartodb-redis'); + +var AnalysisBackend = require('../../lib/cartodb/backends/analysis'); + +describe('analysis-backend limits', function() { + + var redisClient; + var keysToDelete; + var user = 'localhost'; + + beforeEach(function() { + redisClient = redis.createClient(global.environment.redis.port); + keysToDelete = {}; + var redisPool = new RedisPool(global.environment.redis); + this.metadataBackend = cartodbRedis({pool: redisPool}); + }); + + afterEach(function(done) { + redisClient.quit(function() { + testHelper.deleteRedisKeys(keysToDelete, done); + }); + }); + + function withAnalysesLimits(limits, callback) { + redisClient.SELECT(5, function(err) { + if (err) { + return callback(err); + } + var analysesLimitsKey = 'limits:analyses:' + user; + redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) { + if (err) { + return callback(err); + } + keysToDelete[analysesLimitsKey] = 5; + return callback(); + }); + }); + } + + it("should use limits from configuration", function(done) { + var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } }); + analysisBackend.getAnalysesLimits(user, function(err, result) { + assert.ok(!err, err); + + assert.ok(result.analyses.moran); + assert.equal(result.analyses.moran.timeout, 5000); + + assert.ok(result.analyses.kmeans); + assert.equal(result.analyses.kmeans.timeout, 5000); + + done(); + }); + }); + + it("should use limits from redis", function(done) { + var self = this; + var limits = ['moran', 5000]; + + withAnalysesLimits(limits, function(err) { + if (err) { + return done(err); + } + + var analysisBackend = new AnalysisBackend(self.metadataBackend); + analysisBackend.getAnalysesLimits(user, function(err, result) { + assert.ok(!err, err); + + assert.ok(result.analyses.moran); + assert.equal(result.analyses.moran.timeout, 5000); + + done(); + }); + }); + }); + + it("should use limits from redis and configuration, redis takes priority", function(done) { + var self = this; + var limits = ['moran', 5000]; + + withAnalysesLimits(limits, function(err) { + if (err) { + return done(err); + } + + var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } }); + analysisBackend.getAnalysesLimits(user, function(err, result) { + assert.ok(!err, err); + + assert.ok(result.analyses.moran); + assert.equal(result.analyses.moran.timeout, 5000); + + done(); + }); + }); + }); + + it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) { + var self = this; + var limits = ['moran', 5000]; + + withAnalysesLimits(limits, function(err) { + if (err) { + return done(err); + } + + var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } }); + analysisBackend.getAnalysesLimits(user, function(err, result) { + assert.ok(!err, err); + + assert.ok(result.analyses.moran); + assert.equal(result.analyses.moran.timeout, 5000); + + assert.ok(result.analyses.kmeans); + assert.equal(result.analyses.kmeans.timeout, 1000); + + done(); + }); + }); + }); + +});