CartoDB redis interaction delegated to "cartodb-redis" module
This commit is contained in:
Sandro Santilli
parent
e412a0f4b6
commit
a60a3adc12
1
Makefile
1
Makefile
@ -17,7 +17,6 @@ config/environments/test.js: config.status--test
|
||||
|
||||
check-local: config/environments/test.js
|
||||
./run_tests.sh ${RUNTESTFLAGS} \
|
||||
test/unit/cartodb/redis_pool.test.js \
|
||||
test/unit/cartodb/req2params.test.js \
|
||||
test/acceptance/cache_validator.js \
|
||||
test/acceptance/server.js \
|
||||
|
||||
4
NEWS.md
4
NEWS.md
@ -14,6 +14,10 @@ Bug fixes:
|
||||
* Fix http status on database authentication error (windshaft/#94)
|
||||
* Fix text-face-name error at layergroup creation (windshaft/#93)
|
||||
|
||||
Other changes:
|
||||
|
||||
* CartoDB redis interaction delegated to "cartodb-redis" module
|
||||
|
||||
|
||||
1.4.1 -- 2013-11-08
|
||||
-------------------
|
||||
|
||||
@ -1,352 +0,0 @@
|
||||
/**
|
||||
* User: simon
|
||||
* Date: 30/08/2011
|
||||
* Time: 21:10
|
||||
* Desc: CartoDB helper.
|
||||
* Retrieves dbname (based on subdomain/username)
|
||||
* and geometry type from the redis stores of cartodb
|
||||
*/
|
||||
|
||||
var strftime = require('strftime');
|
||||
|
||||
var RedisPool = require("./redis_pool")
|
||||
, _ = require('underscore')
|
||||
, Step = require('step');
|
||||
|
||||
module.exports = function() {
|
||||
var redis_pool = new RedisPool(global.environment.redis);
|
||||
|
||||
|
||||
var me = {
|
||||
user_metadata_db: 5,
|
||||
table_metadata_db: 0,
|
||||
user_key: "rails:users:<%= username %>",
|
||||
table_key: "rails:<%= database_name %>:<%= table_name %>",
|
||||
global_mapview_key: "user:<%= username %>:mapviews:global",
|
||||
tagged_mapview_key: "user:<%= username %>:mapviews:stat_tag:<%= stat_tag %>"
|
||||
};
|
||||
|
||||
me.userFromHostname = function(hostname) {
|
||||
return hostname.split('.')[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the database name for this particular subdomain/username
|
||||
*
|
||||
* @param req - standard express req object. importantly contains host information
|
||||
* @param callback - gets called with args(err, dbname)
|
||||
*/
|
||||
me.getDatabase = function(req, callback) {
|
||||
// strip subdomain from header host
|
||||
var username = me.userFromHostname(req.headers.host);
|
||||
var redisKey = _.template(this.user_key, {username: username});
|
||||
|
||||
this.retrieve(this.user_metadata_db, redisKey, 'database_name', function(err, dbname) {
|
||||
if ( err ) callback(err, null);
|
||||
else if ( dbname === null ) {
|
||||
callback(new Error("missing " + username + "'s database_name in redis (try CARTODB/script/restore_redis)"), null);
|
||||
}
|
||||
else callback(err, dbname);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Increment mapview count for a user
|
||||
*
|
||||
* @param username
|
||||
* @param stat_tag
|
||||
* @param callback will be called with the new value
|
||||
*/
|
||||
me.incMapviewCount = function(username, stat_tag, callback) {
|
||||
var that = this;
|
||||
var now = strftime("%Y%m%d", new Date());
|
||||
var redisKey;
|
||||
Step (
|
||||
function incrementGlobal() {
|
||||
redisKey = _.template(that.global_mapview_key, {username: username});
|
||||
that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
|
||||
},
|
||||
function incrementTag(err, val) {
|
||||
if ( err ) throw err;
|
||||
if ( _.isUndefined(stat_tag) ) return 1;
|
||||
redisKey = _.template(that.tagged_mapview_key, {username: username, stat_tag: stat_tag});
|
||||
that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
|
||||
},
|
||||
function finish(err, val) {
|
||||
if ( callback ) callback(err);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Get the user id for this particular subdomain/username
|
||||
*
|
||||
* @param req - standard express req object. importantly contains host information
|
||||
* @param callback
|
||||
*/
|
||||
me.getId= function(req, callback) {
|
||||
// strip subdomain from header host
|
||||
var username = me.userFromHostname(req.headers.host);
|
||||
var redisKey = _.template(this.user_key, {username: username});
|
||||
|
||||
this.retrieve(this.user_metadata_db, redisKey, 'id', function(err, dbname) {
|
||||
if ( err ) callback(err, null);
|
||||
else if ( dbname === null ) {
|
||||
callback(new Error("missing " + username + "'s id in redis (try CARTODB/script/restore_redis)"), null);
|
||||
}
|
||||
else callback(err, dbname);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Get the database host this particular subdomain/username
|
||||
*
|
||||
* @param req - standard express req object. importantly contains host information
|
||||
* @param callback
|
||||
*/
|
||||
me.getDatabaseHost= function(req, callback) {
|
||||
// strip subdomain from header host
|
||||
var username = me.userFromHostname(req.headers.host);
|
||||
var redisKey = _.template(this.user_key, {username: username});
|
||||
|
||||
this.retrieve(this.user_metadata_db, redisKey, 'database_host', function(err, dbname) {
|
||||
if ( err ) callback(err, null);
|
||||
else {
|
||||
if ( dbname === null ) {
|
||||
/* database_host was introduced in cartodb-2.5.0,
|
||||
* for older versions we'll just use configured host */
|
||||
//console.log("WARNING: missing " + username + "'s database_host in redis (try CARTODB/script/restore_redis)");
|
||||
}
|
||||
callback(err, dbname);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Get the database password for this particular subdomain/username
|
||||
*
|
||||
* @param req - standard express req object. importantly contains host information
|
||||
* @param callback
|
||||
*/
|
||||
me.getDatabasePassword= function(req, callback) {
|
||||
// strip subdomain from header host
|
||||
var username = me.userFromHostname(req.headers.host);
|
||||
var redisKey = _.template(this.user_key, {username: username});
|
||||
|
||||
this.retrieve(this.user_metadata_db, redisKey, 'database_password', function(err, data) {
|
||||
if ( err ) callback(err, null);
|
||||
else {
|
||||
if ( data === null ) {
|
||||
/* database_password was introduced in cartodb-2.5.0,
|
||||
* for older versions we'll just use configured password */
|
||||
//console.log("WARNING: missing " + username + "'s database_password in redis (try CARTODB/script/restore_redis)");
|
||||
}
|
||||
callback(err, data);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Check the user map key for this particular subdomain/username
|
||||
*
|
||||
* @param req - standard express req object. importantly contains host information
|
||||
* @param callback
|
||||
*/
|
||||
me.checkMapKey = function(req, callback) {
|
||||
// strip subdomain from header host
|
||||
var username = me.userFromHostname(req.headers.host);
|
||||
var redisKey = "rails:users:" + username;
|
||||
this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
|
||||
var valid = 0;
|
||||
if ( val ) {
|
||||
if ( val == req.query.map_key ) valid = 1;
|
||||
else if ( val == req.query.api_key ) valid = 1;
|
||||
// check also in request body
|
||||
else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
|
||||
else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
|
||||
}
|
||||
callback(err, valid);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Get privacy for cartodb table
|
||||
*
|
||||
* @param req - standard req object. Importantly contains table and host information
|
||||
* @param callback - is the table private or not?
|
||||
*/
|
||||
me.authorize= function(req, callback) {
|
||||
var that = this;
|
||||
|
||||
Step(
|
||||
function(){
|
||||
that.checkMapKey(req, this);
|
||||
},
|
||||
function checkIfInternal(err, check_result){
|
||||
if (err) throw err;
|
||||
|
||||
// if unauthorized continue to check table privacy
|
||||
if (check_result !== 1) return true;
|
||||
|
||||
// authorized by key, login as db owner
|
||||
var user_params = {};
|
||||
var auth_user = global.settings.postgres_auth_user;
|
||||
var auth_pass = global.settings.postgres_auth_pass;
|
||||
Step(
|
||||
function getId() {
|
||||
that.getId(req, this);
|
||||
},
|
||||
function(err, user_id) {
|
||||
if (err) throw err;
|
||||
user_params['user_id'] = user_id;
|
||||
var dbuser = _.template(auth_user, user_params);
|
||||
_.extend(req.params, {dbuser:dbuser});
|
||||
|
||||
// skip looking up user_password if postgres_auth_pass
|
||||
// doesn't contain the "user_password" label
|
||||
if ( ! auth_pass.match(/\buser_password\b/) ) return null;
|
||||
|
||||
that.getDatabasePassword(req, this);
|
||||
},
|
||||
function(err, user_password) {
|
||||
if (err) throw err;
|
||||
user_params['user_password'] = user_password;
|
||||
if ( auth_pass ) {
|
||||
var dbpass = _.template(auth_pass, user_params);
|
||||
_.extend(req.params, {dbpassword:dbpass});
|
||||
}
|
||||
return true;
|
||||
},
|
||||
function finish(err) {
|
||||
callback(err, true); // authorized (or error)
|
||||
}
|
||||
);
|
||||
}
|
||||
,function getDatabase(err, data){
|
||||
if (err) throw err;
|
||||
that.getDatabase(req, this);
|
||||
},
|
||||
function getPrivacy(err, data){
|
||||
if (err) throw err;
|
||||
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
|
||||
|
||||
that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
|
||||
},
|
||||
function(err, data){
|
||||
callback(err, data);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Get the geometry type for this particular table;
|
||||
* @param req - standard req object. Importantly contains table and host information
|
||||
* @param callback
|
||||
*/
|
||||
me.getGeometryType = function(req, callback){
|
||||
var that = this;
|
||||
|
||||
Step(
|
||||
function(){
|
||||
that.getDatabase(req, this)
|
||||
},
|
||||
function(err, data){
|
||||
if (err) throw err;
|
||||
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
|
||||
|
||||
that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this);
|
||||
},
|
||||
function(err, data){
|
||||
callback(err, data);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
me.getInfowindow = function(req, callback){
|
||||
var that = this;
|
||||
|
||||
Step(
|
||||
function(){
|
||||
that.getDatabase(req, this);
|
||||
},
|
||||
function(err, data) {
|
||||
if (err) throw err;
|
||||
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
|
||||
that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
|
||||
},
|
||||
function(err, data){
|
||||
callback(err, data);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
me.getMapMetadata = function(req, callback){
|
||||
var that = this;
|
||||
|
||||
Step(
|
||||
function(){
|
||||
that.getDatabase(req, this);
|
||||
},
|
||||
function(err, data) {
|
||||
if (err) throw err;
|
||||
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
|
||||
|
||||
that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this);
|
||||
},
|
||||
function(err, data){
|
||||
callback(err, data);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
// Redis Hash lookup
|
||||
// @param callback will be invoked with args (err, reply)
|
||||
// note that reply is null when the key is missing
|
||||
me.retrieve = function(db, redisKey, hashKey, callback) {
|
||||
this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
|
||||
};
|
||||
|
||||
// Redis Set member check
|
||||
me.inSet = function(db, setKey, member, callback) {
|
||||
this.redisCmd(db,'SISMEMBER',[setKey, member], callback);
|
||||
};
|
||||
|
||||
// Redis INCREMENT
|
||||
me.increment = function(db, key, callback) {
|
||||
this.redisCmd(db,'INCR', key, callback);
|
||||
};
|
||||
|
||||
/**
|
||||
* Use Redis
|
||||
*
|
||||
* @param db - redis database number
|
||||
* @param redisFunc - the redis function to execute
|
||||
* @param redisArgs - the arguments for the redis function in an array
|
||||
* @param callback - function to pass results too.
|
||||
*/
|
||||
me.redisCmd = function(db, redisFunc, redisArgs, callback) {
|
||||
var redisClient;
|
||||
|
||||
Step(
|
||||
function getRedisClient() {
|
||||
redis_pool.acquire(db, this);
|
||||
},
|
||||
function executeQuery(err, data) {
|
||||
if ( err ) throw err;
|
||||
redisClient = data;
|
||||
redisArgs.push(this);
|
||||
redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
|
||||
},
|
||||
function releaseRedisClient(err, data) {
|
||||
if ( ! _.isUndefined(redisClient) ) redis_pool.release(db, redisClient);
|
||||
callback(err, data);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
return me;
|
||||
}();
|
||||
@ -1,80 +0,0 @@
|
||||
/**
|
||||
* RedisPool. A database specific redis pooling lib
|
||||
*
|
||||
*/
|
||||
|
||||
var redis = require('redis')
|
||||
, _ = require('underscore')
|
||||
, Pool = require('generic-pool').Pool;
|
||||
|
||||
// constructor.
|
||||
//
|
||||
// - `opts` {Object} optional config for redis and pooling
|
||||
var RedisPool = function(opts){
|
||||
var opts = opts || {};
|
||||
var defaults = {
|
||||
host: '127.0.0.1',
|
||||
port: '6379',
|
||||
max: 50,
|
||||
idleTimeoutMillis: 10000,
|
||||
reapIntervalMillis: 1000,
|
||||
log: false
|
||||
};
|
||||
var options = _.defaults(opts, defaults)
|
||||
|
||||
var me = {
|
||||
pools: {} // cached pools by DB name
|
||||
};
|
||||
|
||||
// Acquire resource.
|
||||
//
|
||||
// - `database` {String} redis database name
|
||||
// - `callback` {Function} callback to call once acquired. Takes the form
|
||||
// `callback(err, resource)`
|
||||
me.acquire = function(database, callback) {
|
||||
if (!this.pools[database]) {
|
||||
this.pools[database] = this.makePool(database);
|
||||
}
|
||||
this.pools[database].acquire(function(err,resource) {
|
||||
callback(err, resource);
|
||||
});
|
||||
};
|
||||
|
||||
// Release resource.
|
||||
//
|
||||
// - `database` {String} redis database name
|
||||
// - `resource` {Object} resource object to release
|
||||
me.release = function(database, resource) {
|
||||
this.pools[database] && this.pools[database].release(resource);
|
||||
};
|
||||
|
||||
// Factory for pool objects.
|
||||
me.makePool = function(database) {
|
||||
return Pool({
|
||||
name: database,
|
||||
create: function(callback){
|
||||
var client = redis.createClient(options.port, options.host);
|
||||
client.on('connect', function () {
|
||||
client.send_anyway = true;
|
||||
client.select(database);
|
||||
client.send_anyway = false;
|
||||
callback(null, client);
|
||||
});
|
||||
client.on('error', function (err) {
|
||||
callback(err, null);
|
||||
});
|
||||
},
|
||||
destroy: function(client) {
|
||||
return client.quit();
|
||||
},
|
||||
max: options.max,
|
||||
idleTimeoutMillis: options.idleTimeoutMillis,
|
||||
reapIntervalMillis: options.reapIntervalMillis,
|
||||
log: options.log
|
||||
});
|
||||
};
|
||||
|
||||
return me;
|
||||
};
|
||||
|
||||
module.exports = RedisPool;
|
||||
@ -1,6 +1,6 @@
|
||||
var _ = require('underscore')
|
||||
, Step = require('step')
|
||||
, cartoData = require('./carto_data')
|
||||
, cartoData = require('cartodb-redis')
|
||||
, Cache = require('./cache_validator')
|
||||
, mapnik = require('mapnik')
|
||||
, crypto = require('crypto')
|
||||
|
||||
42
npm-shrinkwrap.json
generated
42
npm-shrinkwrap.json
generated
@ -45,6 +45,14 @@
|
||||
"mapnik-reference": {
|
||||
"version": "5.0.7"
|
||||
},
|
||||
"hiredis": {
|
||||
"version": "0.1.15",
|
||||
"dependencies": {
|
||||
"bindings": {
|
||||
"version": "1.1.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"millstone": {
|
||||
"version": "0.6.8",
|
||||
"dependencies": {
|
||||
@ -197,6 +205,9 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"generic-pool": {
|
||||
"version": "2.0.4"
|
||||
},
|
||||
"express": {
|
||||
"version": "2.5.11",
|
||||
"dependencies": {
|
||||
@ -258,35 +269,32 @@
|
||||
"step": {
|
||||
"version": "0.0.5"
|
||||
},
|
||||
"generic-pool": {
|
||||
"version": "2.0.4"
|
||||
},
|
||||
"redis": {
|
||||
"version": "0.8.6"
|
||||
},
|
||||
"hiredis": {
|
||||
"version": "0.1.15",
|
||||
"dependencies": {
|
||||
"bindings": {
|
||||
"version": "1.1.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"request": {
|
||||
"version": "2.9.202"
|
||||
},
|
||||
"cartodb-redis": {
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"generic-pool": {
|
||||
"version": "2.0.4"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mapnik": {
|
||||
"version": "0.7.25"
|
||||
},
|
||||
"strftime": {
|
||||
"version": "0.6.2"
|
||||
},
|
||||
"lzma": {
|
||||
"version": "1.2.3"
|
||||
},
|
||||
"semver": {
|
||||
"version": "1.1.4"
|
||||
},
|
||||
"strftime": {
|
||||
"version": "0.6.2"
|
||||
},
|
||||
"redis": {
|
||||
"version": "0.8.6"
|
||||
},
|
||||
"mocha": {
|
||||
"version": "1.2.1",
|
||||
"dependencies": {
|
||||
|
||||
@ -23,16 +23,15 @@
|
||||
"underscore" : "~1.3.3",
|
||||
"windshaft" : "~0.14.3",
|
||||
"step": "0.0.x",
|
||||
"generic-pool": "~2.0.3",
|
||||
"redis": "~0.8.3",
|
||||
"hiredis": "~0.1.14",
|
||||
"request": "2.9.202",
|
||||
"cartodb-redis": "~0.0.1",
|
||||
"mapnik": "~0.7.22",
|
||||
"strftime": "~0.6.0",
|
||||
"lzma": "~1.2.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"mocha": "1.2.1",
|
||||
"redis": "~0.8.3",
|
||||
"strftime": "~0.6.0",
|
||||
"semver": "~1.1.0"
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
var assert = require('../../support/assert')
|
||||
, _ = require('underscore')
|
||||
, RedisPool = require('../../../lib/cartodb/redis_pool')
|
||||
, tests = module.exports = {};
|
||||
|
||||
suite('redis_pool', function() {
|
||||
|
||||
// configure redis pool instance to use in tests
|
||||
var test_opts = require('../../support/config').redis_pool;
|
||||
|
||||
var redis_pool = new RedisPool(test_opts);
|
||||
|
||||
test('RedisPool object exists', function(done){
|
||||
assert.ok(RedisPool);
|
||||
done();
|
||||
});
|
||||
|
||||
test('RedisPool can create new redis_pool objects with default settings', function(done){
|
||||
var redis_pool = new RedisPool();
|
||||
done();
|
||||
});
|
||||
|
||||
test('RedisPool can create new redis_pool objects with specific settings', function(done){
|
||||
var redis_pool = new RedisPool(_.extend({host:'127.0.0.1', port: '6379'}, test_opts));
|
||||
done();
|
||||
});
|
||||
|
||||
|
||||
test('pool object has an acquire function', function(done){
|
||||
var found=false;
|
||||
var functions = _.functions(redis_pool);
|
||||
for (var i=0; i<functions.length; ++i) {
|
||||
if ( functions[i] == 'acquire' ) { found=true; break; }
|
||||
}
|
||||
assert.ok(found);
|
||||
done();
|
||||
});
|
||||
|
||||
test('calling aquire returns a redis client object that can get/set', function(done){
|
||||
redis_pool.acquire(0, function(err, client){
|
||||
if ( err ) { done(err); return; }
|
||||
client.set("key","value");
|
||||
client.get("key", function(err,data){
|
||||
assert.equal(data, "value");
|
||||
redis_pool.release(0, client); // needed to exit tests
|
||||
done();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
test('calling aquire on another DB returns a redis client object that can get/set', function(done){
|
||||
redis_pool.acquire(2, function(err, client){
|
||||
if ( err ) { done(err); return; }
|
||||
client.set("key","value");
|
||||
client.get("key", function(err,data){
|
||||
assert.equal(data, "value");
|
||||
redis_pool.release(2, client); // needed to exit tests
|
||||
done();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
Loading…
Reference in New Issue
Block a user