cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CartoDB redis interaction delegated to "cartodb-redis" module

Browse Source
This commit is contained in:
Sandro Santilli 2013-11-15 18:36:49 +01:00
parent 4a5c9b0eed
commit 03ea51a375
13 changed files with 49 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -1,5 +1,6 @@
1.7.0 - 2013-MM-DD
------------------
* CartoDB redis interaction delegated to "cartodb-redis" module
1.6.3 - 2013-11-10
------------------

23
app/controllers/app.js
View File

@ -29,7 +29,11 @@ var express = require('express')
, zlib = require('zlib')
, util = require('util')
, spawn = require('child_process').spawn
, Meta = require(global.settings.app_root + '/app/models/metadata')
, Meta = require('cartodb-redis')({
host: global.settings.redis_host,
port: global.settings.redis_port
})
// global.settings.app_root + '/app/models/metadata')
, oAuth = require(global.settings.app_root + '/app/models/oauth')
, PSQL = require(global.settings.app_root + '/app/models/psql')
, ApiKeyAuth = require(global.settings.app_root + '/app/models/apikey_auth')
@ -178,18 +182,17 @@ function handleQuery(req, res) {
}
},
function setDBGetUser(err, data) {
if (err) throw err;
if (err) {
// If the database could not be found, the user is non-existant
if ( err.message.match('missing') ) {
err.message = "Sorry, we can't find this CartoDB. Please check that you have entered the correct domain.";
err.http_status = 404;
}
throw err;
}
database = (data === "" || _.isNull(data) || _.isUndefined(data)) ? database : data;
// If the database could not be found, the user is non-existant
if (_.isNull(database)) {
var msg = "Sorry, we can't find this CartoDB. Please check that you have entered the correct domain.";
err = new Error(msg);
err.http_status = 404;
throw err;
}
if(api_key) {
ApiKeyAuth.verifyRequest(req, this);
} else {

91
app/models/apikey_auth.js
View File

@ -2,87 +2,16 @@
* this module allows to auth user using an pregenerated api key
*/
var RedisPool = require("./redis_pool")
var Meta = require("cartodb-redis")({
host: global.settings.redis_host,
port: global.settings.redis_port
})
, _ = require('underscore')
, Step = require('step');
module.exports = (function() {
var me = {
user_metadata_db: 5,
table_metadata_db: 0,
user_key: "rails:users:<%= username %>",
table_key: "rails:<%= database_name %>:<%= table_name %>"
};
me.retrieve = function(db, redisKey, hashKey, callback) {
this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
};
me.inSet = function(db, setKey, member, callback) {
this.redisCmd(db,'SISMEMBER',[setKey, member], callback);
};
/**
* Use Redis
*
* @param db - redis database number
* @param redisFunc - the redis function to execute
* @param redisArgs - the arguments for the redis function in an array
* @param callback - function to pass results too.
*/
me.redisCmd = function(db, redisFunc, redisArgs, callback) {
var redisClient;
Step(
function() {
var step = this;
RedisPool.acquire(db, function(err, _redisClient) {
if ( err ) { step(err); return };
redisClient = _redisClient;
redisArgs.push(step);
redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
});
},
function releaseRedisClient(err, data) {
if ( redisClient ) RedisPool.release(db, redisClient);
callback(err, data);
}
);
};
/**
* Get the user id for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.getId = function(req, callback) {
// strip subdomain from header host
var username = req.headers.host.split('.')[0];
var redisKey = _.template(this.user_key, {username: username});
this.retrieve(this.user_metadata_db, redisKey, 'id', callback);
};
/**
* Get the user map key for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.checkAPIKey= function(req, callback) {
// strip subdomain from header host
var username = req.headers.host.split('.')[0];
var redisKey = "rails:users:" + username;
var api_key = req.query.api_key || req.body.api_key;
this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
var allow = 0;
if ( val && val == api_key ) allow = 1;
callback(err, allow);
});
};
var me = {}
/**
* Get privacy for cartodb table
@ -96,20 +25,20 @@ module.exports = (function() {
Step(
// check api key
function(){
that.checkAPIKey(req, this);
Meta.checkAPIKey(req, this);
},
// get user id or fail
function (err, apikey_valid) {
if ( err ) throw err;
if (apikey_valid) {
that.getId(req, this);
Meta.getId(req, this);
} else {
// no auth
callback(false, null);
callback(null, null);
}
},
function (err, user_id){
if (err) callback(err);
else callback(false, user_id);
callback(err, user_id);
}
);
};

75
app/models/metadata.js
View File

@ -1,75 +0,0 @@
/**
* User: simon
* Date: 23/08/2011
* Time: 21:10
* Desc: retrieves users database_name from the redis metadatabase based on subdomain/username
*/
var RedisPool = require("./redis_pool")
, _ = require('underscore')
, Step = require('step');
module.exports = function() {
var me = {
metadata_database: 5,
user_key: "rails:users:<%= username %>"
};
/**
* Get the database name for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.getDatabase = function(req, callback) {
// strip subdomain from header host
var username = req.headers.host.split('.')[0]
var redisKey = _.template(this.user_key, {username: username});
this.retrieve(redisKey, 'database_name', callback);
};
/**
* Get the user id for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.getId= function(req, callback) {
// strip subdomain from header host
var username = req.headers.host.split('.')[0]
var redisKey = _.template(this.user_key, {username: username});
this.retrieve(redisKey, 'id', callback);
};
/**
* Make a data access call to Redis
*
* @param redisKey - the base redis key where the metadata hash lives
* @param hashKey - the specific metadata you want to retrieve
* @param callback - function to pass metadata too. err,data args
*/
me.retrieve = function(redisKey, hashKey, callback) {
var that = this;
var redisClient;
Step(
function getRedisClient() {
RedisPool.acquire(that.metadata_database, this);
},
function lookupMetadata(err, data) {
if (err) throw err;
redisClient = data;
redisClient.HGET(redisKey, hashKey, this);
},
function releaseRedisClient(err, data) {
if ( redisClient ) RedisPool.release(that.metadata_database, redisClient);
callback(err, data);
}
);
};
return me;
}();

10
app/models/oauth.js
View File

@ -1,5 +1,8 @@
// too bound to the request object, but ok for now
var RedisPool = require("./redis_pool")
var RedisPool = require("../../node_modules/cartodb-redis/lib/redis_pool.js")({
host: global.settings.redis_host,
port: global.settings.redis_port,
})
, _ = require('underscore')
, OAuthUtil = require('oauth-client')
, url = require('url')
@ -137,13 +140,16 @@ var oAuth = function(){
);
};
// TODO: move to cartodb-redis !
me.getOAuthHash = function(access_key, callback){
var that = this;
RedisPool.acquire(this.oauth_database, function(err, client){
if ( err ) { callback(err); return; }
var redisClient = client;
redisClient.HGETALL(_.template(that.oauth_user_key, {oauth_access_key: access_key}), function(err, data){
var key = _.template(that.oauth_user_key, {oauth_access_key: access_key});
redisClient.HGETALL(key, function(err, data){
RedisPool.release(that.oauth_database, redisClient);
if ( ! err && data === null ) data = {};
callback(err, data);
});
});

55
app/models/redis_pool.js
View File

@ -1,55 +0,0 @@
var redis = require('redis')
, Pool = require('generic-pool').Pool;
var RedisPool = {
// Acquire resource.
//
// - `database` {String} redis database name
// - `callback` {Function} callback to call once acquired. Takes the form
// `callback(err, resource)`
acquire: function(database, callback) {
if (!this.pools[database]) {
this.pools[database] = this.makePool(database);
}
this.pools[database].acquire(function(err, resource) {
callback(err, resource);
});
},
// Release resource.
//
// - `database` {String} redis database name
// - `resource` {Object} resource object to release
release: function(database, resource) {
this.pools[database] && this.pools[database].release(resource);
},
// Cache of pools by database name.
pools: {},
// Factory for pool objects.
makePool: function(database) {
return Pool({
name: database,
create: function(callback){
var client = redis.createClient(global.settings.redis_port, global.settings.redis_host);
client.on('connect', function () {
client.send_anyway = true;
client.select(database);
client.send_anyway = false;
});
return callback(client);
},
destroy: function(client) {
return client.quit();
},
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis,
log: global.settings.redisLog
});
}
}
module.exports = RedisPool;

24
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "cartodb_sql_api",
"version": "1.6.0",
"version": "1.7.0",
"dependencies": {
"express": {
"version": "2.5.11",
@ -59,17 +59,14 @@
}
}
},
"generic-pool": {
"version": "2.0.4"
},
"redis": {
"version": "0.7.1"
},
"hiredis": {
"version": "0.1.15",
"cartodb-redis": {
"version": "0.1.0",
"dependencies": {
"bindings": {
"version": "1.1.1"
"strftime": {
"version": "0.6.2"
},
"generic-pool": {
"version": "2.0.4"
}
}
},
@ -103,6 +100,9 @@
"lru-cache": {
"version": "2.2.4"
},
"redis": {
"version": "0.7.1"
},
"zipfile": {
"version": "0.3.4"
},
@ -135,7 +135,7 @@
"version": "1.0.2"
},
"debug": {
"version": "0.7.2"
"version": "0.7.4"
}
}
}

5
package.json
View File

@ -18,9 +18,7 @@
"underscore.string": "~1.1.6",
"pg": "~2.6.2",
"express": "~2.5.11",
"generic-pool": "~2.0.2",
"redis": "0.7.1",
"hiredis": "*",
"cartodb-redis": "~0.1.0",
"step": "0.0.x",
"topojson": "0.0.8",
"oauth-client": "0.2.0",
@ -28,6 +26,7 @@
"lru-cache":"~2.2.2"
},
"devDependencies": {
"redis": "0.7.1",
"mocha": "1.2.1",
"zipfile": "~0.3.2",
"libxmljs": "~0.6.1"

1
test/acceptance/app.auth.test.js
View File

@ -12,7 +12,6 @@ test('valid api key should allow insert in protected tables', function(done){
assert.response(app, {
// view prepare_db.sh to see where to set api_key
url: "/api/v1/sql?api_key=1234&q=INSERT%20INTO%20private_table%20(name)%20VALUES%20('app_auth_test1')",
headers: {host: 'vizzuality.localhost.lan:8080' },
method: 'GET'
},{}, function(res) {

5
test/acceptance/app.test.js
View File

@ -829,9 +829,8 @@ test('GET decent error if domain is incorrect', function(done){
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
headers: {host: 'vizzualinot.cartodb.com'},
method: 'GET'
},{
status: 404
}, function(res){
}, {}, function(res){
assert.equal(res.statusCode, 404, res.statusCode + ( res.statusCode != 200 ? ( ': ' + res.body ) : ''));
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.deepEqual(res.headers['content-disposition'], 'inline');
var result = JSON.parse(res.body);

29
test/unit/metadata.test.js
View File

@ -1,29 +0,0 @@
require('../helper');
var _ = require('underscore')
, redis = require("redis")
, MetaData = require('../../app/models/metadata')
, assert = require('assert')
, tests = module.exports = {};
suite('metadata', function() {
test('test can retrieve database name from header and redis', function(done){
var req = {headers: {host: 'vizzuality.cartodb.com'}};
MetaData.getDatabase(req, function(err, data){
assert.equal(data, 'cartodb_test_user_1_db');
done();
});
});
test('test can retrieve id from header and redis', function(done){
var req = {headers: {host: 'vizzuality.cartodb.com'}};
MetaData.getId(req, function(err, data){
assert.equal(data, '1');
done();
});
});
});

1
test/unit/oauth.test.js
View File

@ -60,6 +60,7 @@ test('test non existant oauth hash for a user based on oauth_token returns empty
var tokens = oAuth.parseTokens(req);
oAuth.getOAuthHash(tokens.oauth_token, function(err, data){
assert.ok(!err, err);
assert.deepEqual(data, {});
done();
});

52
test/unit/redis_pool.test.js
View File

@ -1,52 +0,0 @@
require('../helper');
var _ = require('underscore')
, redis_pool = require('../../app/models/redis_pool')
, assert = require('assert');
suite('redis_pool', function() {
test('test truth', function(){
assert.ok(true, 'it is');
});
test('test can instantiate a RedisPool object', function(){
assert.ok(redis_pool);
});
test('test pool object has an aquire function', function(){
assert.ok(_.includes(_.functions(redis_pool), 'acquire'));
});
test('test calling aquire returns a redis client object that can get/set', function(done){
redis_pool.acquire(0, function(err, client){
assert.ok(!err);
client.set("key","value");
client.get("key", function(err,data){
assert.equal(data, "value");
redis_pool.release(0, client);
done();
});
});
});
test('test calling aquire on another DB returns a redis client object that can get/set', function(done){
redis_pool.acquire("MYDATABASE", function(err, client){
assert.ok(!err);
client.set("key","value");
client.get("key", function(err,data){
assert.equal(data, "value");
redis_pool.release("MYDATABASE", client);
redis_pool.acquire("MYDATABASE", function(err, client){
assert.ok(!err);
client.get("key", function(err,data){
assert.equal(data, "value");
redis_pool.release("MYDATABASE", client);
done();
});
});
})
});
});
});