cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into fix-publisher-connection

Browse Source
This commit is contained in:
Daniel García Aubert 2016-07-07 16:07:41 +02:00
commit ccff602bbf
10 changed files with 266 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
NEWS.md
View File

@ -1,3 +1,17 @@
1.33.1 - 2016-mm-dd
-------------------
New features:
* Allow to setup more than one domain to validate oauth against.
1.33.0 - 2016-07-01
-------------------
New features:
* Add `<%= job_id %>` template support for onerror and onsuccess fallback queries.
1.32.0 - 2016-06-30
-------------------

82
app/auth/oauth.js
View File

@ -3,6 +3,8 @@ var _ = require('underscore');
var OAuthUtil = require('oauth-client');
var step = require('step');
var assert = require('assert');
var CdbRequest = require('../models/cartodb_request');
var cdbReq = new CdbRequest();
var oAuth = (function(){
var me = {
@ -60,79 +62,87 @@ var oAuth = (function(){
return removed;
};
me.getAllowedHosts= function() {
var oauthConfig = global.settings.oauth || {};
return oauthConfig.allowedHosts || ['carto.com', 'cartodb.com'];
};
// do new fancy get User ID
me.verifyRequest = function(req, metadataBackend, callback) {
var that = this;
//TODO: review this
var httpProto = req.protocol;
var passed_tokens;
var ohash;
if(!httpProto || (httpProto !== 'http' && httpProto !== 'https')) {
var msg = "Unknown HTTP protocol " + httpProto + ".";
var unknownProtocolErr = new Error(msg);
unknownProtocolErr.http_status = 500;
return callback(unknownProtocolErr);
}
var username = cdbReq.userByReq(req);
var requestTokens;
var signature;
step(
function getTokensFromURL(){
return oAuth.parseTokens(req);
},
function getOAuthHash(err, data){
function getOAuthHash(err, _requestTokens) {
assert.ifError(err);
// this is oauth request only if oauth headers are present
this.is_oauth_request = !_.isEmpty(data);
this.is_oauth_request = !_.isEmpty(_requestTokens);
if (this.is_oauth_request) {
passed_tokens = data;
that.getOAuthHash(metadataBackend, passed_tokens.oauth_token, this);
requestTokens = _requestTokens;
that.getOAuthHash(metadataBackend, requestTokens.oauth_token, this);
} else {
return null;
}
},
function regenerateSignature(err, data){
function regenerateSignature(err, oAuthHash){
assert.ifError(err);
if (!this.is_oauth_request) {
return null;
}
ohash = data;
var consumer = OAuthUtil.createConsumer(ohash.consumer_key, ohash.consumer_secret);
var access_token = OAuthUtil.createToken(ohash.access_token_token, ohash.access_token_secret);
var consumer = OAuthUtil.createConsumer(oAuthHash.consumer_key, oAuthHash.consumer_secret);
var access_token = OAuthUtil.createToken(oAuthHash.access_token_token, oAuthHash.access_token_secret);
var signer = OAuthUtil.createHmac(consumer, access_token);
var method = req.method;
var host = req.headers.host;
var hostsToValidate = {};
var requestHost = req.headers.host;
hostsToValidate[requestHost] = true;
that.getAllowedHosts().forEach(function(allowedHost) {
hostsToValidate[username + '.' + allowedHost] = true;
});
if(!httpProto || (httpProto !== 'http' && httpProto !== 'https')) {
var msg = "Unknown HTTP protocol " + httpProto + ".";
err = new Error(msg);
err.http_status = 500;
callback(err);
return;
}
var path = httpProto + '://' + host + req.path;
that.splitParams(req.query);
// remove signature from passed_tokens
signature = passed_tokens.oauth_signature;
delete passed_tokens.oauth_signature;
var joined = {};
// remove oauth_signature from body
if(req.body) {
delete req.body.oauth_signature;
}
_.extend(joined, req.body ? req.body : null);
_.extend(joined, passed_tokens);
_.extend(joined, req.query);
signature = requestTokens.oauth_signature;
// remove signature from requestTokens
delete requestTokens.oauth_signature;
var requestParams = _.extend({}, req.body, requestTokens, req.query);
return signer.sign(method, path, joined);
var hosts = Object.keys(hostsToValidate);
var requestSignatures = hosts.map(function(host) {
var url = httpProto + '://' + host + req.path;
return signer.sign(method, url, requestParams);
});
return requestSignatures.reduce(function(validSignature, requestSignature) {
if (signature === requestSignature && !_.isUndefined(requestSignature)) {
validSignature = true;
}
return validSignature;
}, false);
},
function checkSignature(err, data){
assert.ifError(err);
//console.log(data + " should equal the provided signature: " + signature);
callback(err, (signature === data && !_.isUndefined(data)) ? true : null);
function finishValidation(err, hasValidSignature) {
return callback(err, hasValidSignature || null);
}
);
};

7
batch/models/query/fallback.js
View File

@ -30,7 +30,11 @@ Fallback.prototype.getNextQuery = function (job) {
Fallback.prototype.getOnSuccess = function (job) {
if (job.query.query[this.index].status === jobStatus.DONE &&
job.query.query[this.index].fallback_status === jobStatus.PENDING) {
return job.query.query[this.index].onsuccess;
var onsuccessQuery = job.query.query[this.index].onsuccess;
if (onsuccessQuery) {
onsuccessQuery = onsuccessQuery.replace(/<%=\s*job_id\s*%>/g, job.job_id);
}
return onsuccessQuery;
}
};
@ -43,6 +47,7 @@ Fallback.prototype.getOnError = function (job) {
job.query.query[this.index].fallback_status === jobStatus.PENDING) {
var onerrorQuery = job.query.query[this.index].onerror;
if (onerrorQuery) {
onerrorQuery = onerrorQuery.replace(/<%=\s*job_id\s*%>/g, job.job_id);
onerrorQuery = onerrorQuery.replace(/<%=\s*error_message\s*%>/g, job.query.query[this.index].failed_reason);
}
return onerrorQuery;

3
config/environments/production.js.example
View File

@ -85,4 +85,7 @@ module.exports.health = {
username: 'development',
query: 'select 1'
};
module.exports.oauth = {
allowedHosts: ['carto.com', 'cartodb.com']
};
module.exports.disabled_file = 'pids/disabled';

3
config/environments/test.js.example
View File

@ -74,4 +74,7 @@ module.exports.health = {
username: 'vizzuality',
query: 'select 1'
};
module.exports.oauth = {
allowedHosts: ['localhost.lan:8080', 'localhostdb.lan:8080']
};
module.exports.disabled_file = 'pids/disabled';

2
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "cartodb_sql_api",
"version": "1.32.0",
"version": "1.33.1",
"dependencies": {
"cartodb-psql": {
"version": "0.6.1",

2
package.json
View File

@ -5,7 +5,7 @@
"keywords": [
"cartodb"
],
"version": "1.32.0",
"version": "1.33.1",
"repository": {
"type": "git",
"url": "git://github.com/CartoDB/CartoDB-SQL-API.git"

112
test/acceptance/job.error-template.test.js → test/acceptance/job.callback-template.test.js
View File

@ -14,7 +14,7 @@ var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var jobStatus = require('../../batch/job_status');
describe('Batch API query timing', function () {
describe('Batch API callback templates', function () {
function createJob(jobDefinition, callback) {
assert.response(app, {
@ -74,7 +74,13 @@ describe('Batch API query timing', function () {
var expectedQuery = expected.query[index];
assert.ok(expectedQuery);
Object.keys(expectedQuery).forEach(function(expectedKey) {
assert.equal(actualQuery[expectedKey], expectedQuery[expectedKey]);
assert.equal(
actualQuery[expectedKey],
expectedQuery[expectedKey],
'Expected value for key "' + expectedKey + '" does not match: ' + actualQuery[expectedKey] + ' ==' +
expectedQuery[expectedKey] + ' at query index=' + index + '. Full response: ' +
JSON.stringify(actual, null, 4)
);
});
var propsToCheckDate = ['started_at', 'ended_at'];
propsToCheckDate.forEach(function(propToCheckDate) {
@ -101,37 +107,36 @@ describe('Batch API query timing', function () {
});
});
describe('should report start and end time for each query with fallback queries', function () {
describe.skip('should use templates for error_message and job_id onerror callback', function () {
var jobResponse;
before(function(done) {
createJob({
"query": {
"query": [
{
"query": "create table batch_errors (error_message text)"
},
{
"query": "SELECT * FROM invalid_table",
"onerror": "INSERT INTO batch_errors values ('<%= error_message %>')"
}
]
getQueryResult('create table test_batch_errors (job_id text, error_message text)', function(err) {
if (err) {
return done(err);
}
}, function(err, job) {
jobResponse = job;
return done(err);
createJob({
"query": {
"query": [
{
"query": "SELECT * FROM invalid_table",
"onerror": "INSERT INTO test_batch_errors " +
"values ('<%= job_id %>', '<%= error_message %>')"
}
]
}
}, function(err, job) {
jobResponse = job;
return done(err);
});
});
});
it('should keep the original templated query but use the error message', function (done) {
var expectedQuery = {
query: [
{
"query": "create table batch_errors (error_message text)",
status: 'done'
},
{
"query": "SELECT * FROM invalid_table",
"onerror": "INSERT INTO batch_errors values ('<%= error_message %>')",
"onerror": "INSERT INTO test_batch_errors values ('<%= job_id %>', '<%= error_message %>')",
status: 'failed',
fallback_status: 'done'
}
@ -143,12 +148,13 @@ describe('Batch API query timing', function () {
if (job.status === jobStatus.FAILED) {
clearInterval(interval);
validateExpectedResponse(job.query, expectedQuery);
getQueryResult('select * from batch_errors', function(err, result) {
getQueryResult('select * from test_batch_errors', function(err, result) {
if (err) {
return done(err);
}
assert.equal(result.rows[0].job_id, jobResponse.job_id);
assert.equal(result.rows[0].error_message, 'relation "invalid_table" does not exist');
getQueryResult('drop table batch_errors', done);
getQueryResult('drop table test_batch_errors', done);
});
} else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) {
clearInterval(interval);
@ -159,4 +165,62 @@ describe('Batch API query timing', function () {
});
});
describe('should use template for job_id onsuccess callback', function () {
var jobResponse;
before(function(done) {
createJob({
"query": {
"query": [
{
query: "create table batch_jobs (job_id text)"
},
{
"query": "SELECT 1",
"onsuccess": "INSERT INTO batch_jobs values ('<%= job_id %>')"
}
]
}
}, function(err, job) {
jobResponse = job;
return done(err);
});
});
it('should keep the original templated query but use the job_id', function (done) {
var expectedQuery = {
query: [
{
query: "create table batch_jobs (job_id text)",
status: 'done'
},
{
query: "SELECT 1",
onsuccess: "INSERT INTO batch_jobs values ('<%= job_id %>')",
status: 'done',
fallback_status: 'done'
}
]
};
var interval = setInterval(function () {
getJobStatus(jobResponse.job_id, function(err, job) {
if (job.status === jobStatus.DONE) {
clearInterval(interval);
validateExpectedResponse(job.query, expectedQuery);
getQueryResult('select * from batch_jobs', function(err, result) {
if (err) {
return done(err);
}
assert.equal(result.rows[0].job_id, jobResponse.job_id);
getQueryResult('drop table batch_jobs', done);
});
} else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) {
clearInterval(interval);
done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be "done"'));
}
});
}, 50);
});
});
});

159
test/acceptance/job.fallback.test.js
View File

@ -1636,6 +1636,85 @@ describe('Batch API fallback job', function () {
});
});
describe('should fail first "onerror" and job "onerror" and skip the other ones', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'host': 'vizzuality.cartodb.com'
},
method: 'POST',
data: querystring.stringify({
"query": {
"query": [{
"query": "SELECT * FROM atm_madrid limit 1, should fail",
"onerror": "SELECT * FROM atm_madrid limit 2"
}, {
"query": "SELECT * FROM atm_madrid limit 3",
"onerror": "SELECT * FROM atm_madrid limit 4"
}],
"onerror": "SELECT * FROM atm_madrid limit 5"
}
})
}, {
status: 201
}, function (res, err) {
if (err) {
return done(err);
}
fallbackJob = JSON.parse(res.body);
done();
});
});
it('job should fail', function (done) {
var expectedQuery = {
query: [{
query: 'SELECT * FROM atm_madrid limit 1, should fail',
onerror: 'SELECT * FROM atm_madrid limit 2',
status: 'failed',
fallback_status: 'failed',
failed_reason: 'relation "atm_madrid" does not exist'
}, {
query: 'SELECT * FROM atm_madrid limit 3',
onerror: 'SELECT * FROM atm_madrid limit 4',
status: 'skipped',
fallback_status: 'skipped'
}],
onerror: 'SELECT * FROM atm_madrid limit 5'
};
var interval = setInterval(function () {
assert.response(app, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'host': 'vizzuality.cartodb.com'
},
method: 'GET'
}, {
status: 200
}, function (res, err) {
if (err) {
return done(err);
}
var job = JSON.parse(res.body);
if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.FAILED) {
clearInterval(interval);
validateExpectedResponse(job.query, expectedQuery);
done();
} else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) {
clearInterval(interval);
done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed'));
}
});
}, 50);
});
});
describe('should run first "onerror" and job "onerror" and skip the other ones', function () {
var fallbackJob = {};
@ -1717,84 +1796,4 @@ describe('Batch API fallback job', function () {
}, 50);
});
});
describe('should fail first "onerror" and job "onerror" and skip the other ones', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'host': 'vizzuality.cartodb.com'
},
method: 'POST',
data: querystring.stringify({
"query": {
"query": [{
"query": "SELECT * FROM atm_madrid limit 1, should fail",
"onerror": "SELECT * FROM atm_madrid limit 2"
}, {
"query": "SELECT * FROM atm_madrid limit 3",
"onerror": "SELECT * FROM atm_madrid limit 4"
}],
"onerror": "SELECT * FROM atm_madrid limit 5"
}
})
}, {
status: 201
}, function (res, err) {
if (err) {
return done(err);
}
fallbackJob = JSON.parse(res.body);
done();
});
});
it('job should fail', function (done) {
var expectedQuery = {
query: [{
query: 'SELECT * FROM atm_madrid limit 1, should fail',
onerror: 'SELECT * FROM atm_madrid limit 2',
status: 'failed',
fallback_status: 'failed',
failed_reason: 'relation "atm_madrid" does not exist'
}, {
query: 'SELECT * FROM atm_madrid limit 3',
onerror: 'SELECT * FROM atm_madrid limit 4',
status: 'skipped',
fallback_status: 'skipped'
}],
onerror: 'SELECT * FROM atm_madrid limit 5'
};
var interval = setInterval(function () {
assert.response(app, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'host': 'vizzuality.cartodb.com'
},
method: 'GET'
}, {
status: 200
}, function (res, err) {
if (err) {
return done(err);
}
var job = JSON.parse(res.body);
if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.FAILED) {
clearInterval(interval);
validateExpectedResponse(job.query, expectedQuery);
done();
} else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) {
clearInterval(interval);
done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed'));
}
});
}, 50);
});
});
});

26
test/unit/oauth.test.js
View File

@ -78,7 +78,7 @@ it('test can access oauth hash for a user based on access token (oauth_token)',
});
it('test non existant oauth hash for a user based on oauth_token returns empty hash', function(done){
var req = {query:{}, headers:{authorization:full_oauth_header}};
var req = {query:{}, params: { user: 'vizzuality' }, headers:{authorization:full_oauth_header}};
var tokens = oAuth.parseTokens(req);
oAuth.getOAuthHash(metadataBackend, tokens.oauth_token, function(err, data){
@ -91,6 +91,7 @@ it('test non existant oauth hash for a user based on oauth_token returns empty h
it('can return user for verified signature', function(done){
var req = {query:{},
headers:{authorization:real_oauth_header, host: 'vizzuality.testhost.lan' },
params: { user: 'vizzuality' },
protocol: 'http',
method: 'GET',
path: '/api/v1/tables'
@ -103,9 +104,31 @@ it('can return user for verified signature', function(done){
});
});
it('can return user for verified signature (for other allowed domains)', function(done){
var oAuthGetAllowedHostsFn = oAuth.getAllowedHosts;
oAuth.getAllowedHosts = function() {
return ['testhost.lan', 'testhostdb.lan'];
};
var req = {query:{},
headers:{authorization:real_oauth_header, host: 'vizzuality.testhostdb.lan' },
params: { user: 'vizzuality' },
protocol: 'http',
method: 'GET',
path: '/api/v1/tables'
};
oAuth.verifyRequest(req, metadataBackend, function(err, data){
oAuth.getAllowedHosts = oAuthGetAllowedHostsFn;
assert.ok(!err, err);
assert.equal(data, 1);
done();
});
});
it('returns null user for unverified signatures', function(done){
var req = {query:{},
headers:{authorization:real_oauth_header, host: 'vizzuality.testyhost.lan' },
params: { user: 'vizzuality' },
protocol: 'http',
method: 'GET',
path: '/api/v1/tables'
@ -121,6 +144,7 @@ it('returns null user for no oauth', function(done){
var req = {
query:{},
headers:{},
params: { user: 'vizzuality' },
protocol: 'http',
method: 'GET',
path: '/api/v1/tables'