cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #224 from CartoDB/one-redis-mpool

Browse Source 
Use only one redis-mpool across the whole app
This commit is contained in:
Raul Ochoa 2014-10-15 16:31:02 +02:00
commit e52f583e20
10 changed files with 74 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
NEWS.md
View File

@ -1,6 +1,12 @@
1.19.1 -- 2014-mm-dd
--------------------
New features:
- Report to statsd the status of redis pools
Enhancements:
- Share one redis-mpool across the application
1.19.0 -- 2014-10-14
--------------------

21
app.js
View File

@ -8,7 +8,8 @@
*/
var path = require('path'),
fs = require('fs')
fs = require('fs'),
RedisPool = require('redis-mpool')
;
@ -31,7 +32,7 @@ var _ = require('underscore');
global.environment = require(__dirname + '/config/environments/' + ENV);
global.environment.api_hostname = require('os').hostname().split('.')[0];
global.log4js = require('log4js')
global.log4js = require('log4js');
log4js_config = {
appenders: [],
replaceConsole:true
@ -69,13 +70,25 @@ if ( global.environment.rollbar ) {
log4js.configure(log4js_config, { cwd: __dirname });
global.logger = log4js.getLogger();
var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
redisPool = new RedisPool(redisOpts);
// Include cartodb_windshaft only _after_ the "global" variable is set
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
var serverOptions = require('./lib/cartodb/server_options')();
var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
serverOptions = require('./lib/cartodb/server_options')(redisPool);
ws = CartodbWindshaft(serverOptions);
if (global.statsClient) {
redisPool.on('status', function(status) {
var keyPrefix = status.name + '.db' + status.db + '.';
global.statsClient.gauge(keyPrefix + 'count', status.count);
global.statsClient.gauge(keyPrefix + 'unused', status.unused);
global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
});
}
// Maximum number of connections for one process
// 128 is a good number if you have up to 1024 filedescriptors
// 4 is good if you have max 32 filedescriptors

3
config/environments/development.js.example
View File

@ -112,6 +112,9 @@ var config = {
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
},
emitter: {
statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
}
}
,sqlapi: {

3
config/environments/production.js.example
View File

@ -106,6 +106,9 @@ var config = {
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
},
emitter: {
statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
}
}
,sqlapi: {

3
config/environments/staging.js.example
View File

@ -106,6 +106,9 @@ var config = {
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
},
emitter: {
statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
}
}
,sqlapi: {

3
config/environments/test.js.example
View File

@ -106,6 +106,9 @@ var config = {
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
},
emitter: {
statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
}
}
,sqlapi: {

23
lib/cartodb/cartodb_windshaft.js
View File

@ -1,10 +1,6 @@
var _ = require('underscore')
, Step = require('step')
, Windshaft = require('windshaft')
, redisPool = require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}))
// TODO: instanciate cartoData with redisPool
, cartoData = require('cartodb-redis')(global.environment.redis)
, SignedMaps = require('./signed_maps.js')
, TemplateMaps = require('./template_maps.js')
, Cache = require('./cache_validator')
@ -26,6 +22,11 @@ var CartodbWindshaft = function(serverOptions) {
}
}
var redisPool = serverOptions.redis.pool
|| require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}));
var cartoData = require('cartodb-redis')({pool: redisPool});
if(serverOptions.cache_enabled) {
console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
@ -41,7 +42,7 @@ var CartodbWindshaft = function(serverOptions) {
err = new Error("map state cannot be changed by unauthenticated request!");
}
callback(err, req);
}
};
// This is for Templated maps
//
@ -52,7 +53,7 @@ var CartodbWindshaft = function(serverOptions) {
serverOptions.signedMaps = new SignedMaps(redisPool);
var templateMapsOpts = {
max_user_templates: global.environment.maxUserTemplates
}
};
var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps, templateMapsOpts);
// boot
@ -64,7 +65,7 @@ var CartodbWindshaft = function(serverOptions) {
var version = wsversion();
version.windshaft_cartodb = require('../../package.json').version;
return version;
}
};
var ws_sendResponse = ws.sendResponse;
// GET routes for which we don't want to request any caching.
@ -202,7 +203,7 @@ var CartodbWindshaft = function(serverOptions) {
ws.userByReq = function(req) {
return serverOptions.userByReq(req);
}
};
// Add a template
ws.post(template_baseurl, function(req, res) {
@ -445,7 +446,7 @@ var CartodbWindshaft = function(serverOptions) {
function prepareResponse(err, tpl_ids){
if ( err ) throw err;
// NOTE: might omit "cbduser" if == dbowner ...
var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; })
var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; });
return { template_ids: ids };
},
function finish(err, response){
@ -500,7 +501,7 @@ var CartodbWindshaft = function(serverOptions) {
if ( tpl_id[0] && tpl_id[0] != cdbuser ) {
var err = new Error('Cannot instanciate map of user "'
+ tpl_id[0] + '" on database of user "'
+ cdbuser + '"')
+ cdbuser + '"');
err.http_status = 403;
callback(err);
return;
@ -665,6 +666,6 @@ var CartodbWindshaft = function(serverOptions) {
// ---- Template maps interface ends @}
return ws;
}
};
module.exports = CartodbWindshaft;

16
lib/cartodb/server_options.js
View File

@ -1,10 +1,9 @@
var _ = require('underscore')
, Step = require('step')
, cartoData = require('cartodb-redis')(global.environment.redis)
, Cache = require('./cache_validator')
, QueryTablesApi = require('./api/query_tables_api')
, crypto = require('crypto')
, LZMA = require('lzma').LZMA;
, LZMA = require('lzma').LZMA
;
// This is for backward compatibility with 1.3.3
@ -16,11 +15,11 @@ if ( _.isUndefined(global.environment.sqlapi.domain) ) {
}
}
module.exports = function(){
var lzmaWorker = new LZMA();
var queryTablesApi = new QueryTablesApi();
module.exports = function(redisPool) {
var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis;
var cartoData = require('cartodb-redis')(redisOpts),
lzmaWorker = new LZMA(),
queryTablesApi = new QueryTablesApi();
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
@ -77,6 +76,9 @@ module.exports = function(){
// See http://github.com/CartoDB/Windshaft-cartodb/issues/161
me.redis.unwatchOnRelease = false;
// Re-use redisPool
me.redis.pool = redisPool;
/* This whole block is about generating X-Cache-Channel { */
// TODO: review lifetime of elements of this cache

27
npm-shrinkwrap.json generated
View File

@ -70,7 +70,8 @@
"dependencies": {
"async": {
"version": "0.2.10",
"from": "async@~0.2.0"
"from": "async@~0.2.0",
"resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz"
},
"readable-stream": {
"version": "1.0.33-1",
@ -90,7 +91,7 @@
},
"inherits": {
"version": "2.0.1",
"from": "inherits@2"
"from": "inherits@~2.0.1"
}
}
}
@ -106,9 +107,9 @@
"resolved": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0"
},
"redis-mpool": {
"version": "0.1.0",
"from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
"resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
"version": "0.2.0",
"from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.2.0",
"resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.2.0",
"dependencies": {
"generic-pool": {
"version": "2.1.1",
@ -162,12 +163,13 @@
},
"underscore": {
"version": "1.6.0",
"from": "underscore@~1.6.0"
"from": "underscore@~1.6.0",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
},
"windshaft": {
"version": "0.28.2",
"from": "https://github.com/CartoDB/Windshaft/tarball/0.28.2",
"resolved": "https://github.com/CartoDB/Windshaft/tarball/0.28.2",
"version": "0.29.0",
"from": "https://github.com/CartoDB/Windshaft/tarball/0.29.0",
"resolved": "https://github.com/CartoDB/Windshaft/tarball/0.29.0",
"dependencies": {
"chronograph": {
"version": "0.1.0",
@ -272,7 +274,8 @@
},
"async": {
"version": "0.9.0",
"from": "async@~0.9.0"
"from": "async@~0.9.0",
"resolved": "https://registry.npmjs.org/async/-/async-0.9.0.tgz"
}
}
},
@ -1462,6 +1465,7 @@
"express": {
"version": "2.5.11",
"from": "express@~2.5.11",
"resolved": "https://registry.npmjs.org/express/-/express-2.5.11.tgz",
"dependencies": {
"connect": {
"version": "1.9.2",
@ -1469,7 +1473,8 @@
"dependencies": {
"formidable": {
"version": "1.0.15",
"from": "formidable@1.0.x"
"from": "formidable@1.0.x",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.15.tgz"
}
}
},

4
package.json
View File

@ -25,12 +25,12 @@
"node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
"underscore" : "~1.6.0",
"dot": "~1.0.2",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.28.2",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.29.0",
"step": "~0.0.5",
"request": "~2.9.203",
"cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
"cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
"redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
"redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.2.0",
"lzma": "~1.3.7",
"log4js": "~0.6.17",
"rollbar": "~0.3.13"