cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Starts using cartodb-psql node module in SQL API

Browse Source
This commit is contained in:
Raul Ochoa 2014-08-11 20:15:55 +02:00
parent 951636892c
commit 77cb86154c
8 changed files with 7 additions and 650 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/controllers/app.js
View File

@ -33,8 +33,7 @@ var express = require('express')
, Profiler = require('step-profiler')
, StatsD = require('node-statsd').StatsD
, MetadataDB = require('cartodb-redis')
, PSQL = require(global.settings.app_root + '/app/models/psql')
, PSQLWrapper = require(global.settings.app_root + '/app/sql/psql_wrapper')
, PSQL = require('cartodb-psql')
, CdbRequest = require(global.settings.app_root + '/app/models/cartodb_request')
, AuthApi = require(global.settings.app_root + '/app/auth/auth_api')
, _ = require('underscore')
@ -351,7 +350,7 @@ function handleQuery(req, res) {
return false;
} else {
//TODO: sanitize cdbuser
console.log("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$");
console.log("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)");
pg.query("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)", function (err, result) {
if (err) {
self(err);
@ -446,7 +445,7 @@ function handleQuery(req, res) {
checkAborted('generateFormat');
// TODO: drop this, fix UI!
sql = new PSQLWrapper(sql).orderBy(orderBy, sortOrder).window(limit, offset).query();
sql = new PSQL.QueryWrapper(sql).orderBy(orderBy, sortOrder).window(limit, offset).query();
var opts = {
dbopts: dbopts,

2
app/models/formats/ogr.js
View File

@ -2,7 +2,7 @@ var crypto = require('crypto'),
Step = require('step'),
fs = require('fs'),
_ = require('underscore'),
PSQL = require(global.settings.app_root + '/app/models/psql'),
PSQL = require('cartodb-psql'),
spawn = require('child_process').spawn;
// Keeps track of what's waiting baking for export

2
app/models/formats/pg.js
View File

@ -1,5 +1,5 @@
var Step = require('step'),
PSQL = require(global.settings.app_root + '/app/models/psql');
PSQL = require('cartodb-psql');
function PostgresFormat(id) {
this.id = id;

284
app/models/psql.js
View File

@ -1,284 +0,0 @@
var _ = require('underscore'),
PSQLWrapper = require('../sql/psql_wrapper'),
Step = require('step'),
pg = require('pg');//.native; // disabled for now due to: https://github.com/brianc/node-postgres/issues/48
_.mixin(require('underscore.string'));
// Max database connections in the pool
// Subsequent connections will block waiting for a free slot
pg.defaults.poolSize = global.settings.db_pool_size || 16;
// Milliseconds of idle time before removing connection from pool
pg.defaults.poolIdleTimeout = global.settings.db_pool_idleTimeout || 30000;
// Frequency to check for idle clients within the pool, ms
pg.defaults.reapIntervalMillis = global.settings.db_pool_reapInterval || 1000;
pg.on('error', function(err, client) {
console.log("PostgreSQL connection error: " + err);
});
// Workaround for https://github.com/Vizzuality/CartoDB-SQL-API/issues/100
var types = require(__dirname + '/../../node_modules/pg/lib/types');
var arrayParser = require(__dirname + '/../../node_modules/pg/lib/types/arrayParser');
var floatParser = function(val) {
return parseFloat(val);
};
var floatArrayParser = function(val) {
if(!val) { return null; }
var p = arrayParser.create(val, function(entry) {
return floatParser(entry);
});
return p.parse();
};
types.setTypeParser(20, floatParser); // int8
types.setTypeParser(700, floatParser); // float4
types.setTypeParser(701, floatParser); // float8
types.setTypeParser(1700, floatParser); // numeric
types.setTypeParser(1021, floatArrayParser); // _float4
types.setTypeParser(1022, floatArrayParser); // _float8
types.setTypeParser(1231, floatArrayParser); // _numeric
types.setTypeParser(1016, floatArrayParser); // _int8
// Standard type->name mappnig (up to oid=2000)
var stdTypeName = {
16: 'bool',
17: 'bytea',
20: 'int8',
21: 'int2',
23: 'int4',
25: 'text',
26: 'oid',
114: 'JSON',
700: 'float4',
701: 'float8',
1000: '_bool',
1015: '_varchar',
1042: 'bpchar',
1043: 'varchar',
1005: '_int2',
1007: '_int4',
1014: '_bpchar',
1016: '_int8',
1021: '_float4',
1022: '_float8',
1008: '_regproc',
1009: '_text',
1082: 'date',
1114: 'timestamp',
1182: '_date',
1184: 'timestampz',
1186: 'interval',
1231: '_numeric',
1700: 'numeric'
};
// Holds a typeId->typeName mapping for each
// database ever connected to
var extTypeName = {};
// PSQL
//
// A simple postgres wrapper with logic about username and database to connect
//
// * intended for use with pg_bouncer
// * defaults to connecting with a "READ ONLY" user to given DB if not passed a specific user_id
//
// @param opts connection options:
// user: database username
// pass: database user password
// host: database host
// port: database port
// dbname: database name
//
var PSQL = function(dbopts) {
var error_text = "Incorrect access parameters. If you are accessing via OAuth, please check your tokens are correct. For public users, please ensure your table is published."
if ( ! dbopts || ( !_.isString(dbopts.user) && !_.isString(dbopts.dbname)))
{
// console.log("DBOPTS: "); console.dir(dbopts);
throw new Error(error_text);
}
var me = {
dbopts: dbopts
};
me.username = function(){
return this.dbopts.user;
};
me.password = function(){
return this.dbopts.pass;
};
me.database = function(){
return this.dbopts.dbname;
};
me.dbhost = function(){
return this.dbopts.host;
};
me.dbport = function(){
return this.dbopts.port;
};
me.conString = "tcp://" + me.username() +
":" + me.password() + // this line only if not-null ?
"@" +
me.dbhost() + ":" +
me.dbport() + "/" +
me.database();
me.dbkey = function(){
return this.database(); // + ":" + this.dbhost() + ":" + me.dbport();
};
me.ensureTypeCache = function(cb) {
var db = this.dbkey();
if ( extTypeName[db] ) { cb(); return; }
pg.connect(this.conString, function(err, client, done) {
if ( err ) { cb(err); return; }
var types = ["'geometry'","'raster'"]; // types of interest
client.query("SELECT oid, typname FROM pg_type where typname in (" + types.join(',') + ")", function(err,res) {
done();
if ( err ) { cb(err); return; }
var cache = {};
res.rows.map(function(r) {
cache[r.oid] = r.typname;
});
extTypeName[db] = cache;
cb();
});
});
}
// Return type name for a type identifier
//
// Possibly returns undefined, for unkonwn (uncached)
//
me.typeName = function(typeId) {
return stdTypeName[typeId] ? stdTypeName[typeId] : extTypeName[this.dbkey()][typeId];
}
me.connect = function(cb){
var that = this;
this.ensureTypeCache(function(err) {
if ( err ) cb(err);
else pg.connect(that.conString, cb);
});
};
me.eventedQuery = function(sql, callback){
var that = this;
Step(
function(){
that.sanitize(sql, this);
},
function(err, clean){
if (err) throw err;
that.connect(this);
},
function(err, client, done){
var next = this;
if (err) throw err;
var query = client.query(sql);
// forward notices to query
var noticeListener = function() {
query.emit('notice', arguments);
};
client.on('notice', noticeListener);
// NOTE: for some obscure reason passing "done" directly
// as the listener works but can be slower
// (by x2 factor!)
query.on('end', function() {
client.removeListener('notice', noticeListener);
done();
});
next(null, query, client);
},
function(err, query, client){
var queryCanceller = function() {
pg.cancel(undefined, client, query);
};
callback(err, query, queryCanceller);
}
);
};
me.quoteIdentifier = function(str) {
return pg.Client.prototype.escapeIdentifier(str);
};
me.escapeLiteral = function(s) {
return pg.Client.prototype.escapeLiteral(str);
};
me.query = function(sql, callback){
var that = this;
var finish;
Step(
function(){
that.sanitize(sql, this);
},
function(err, clean){
if (err) throw err;
that.connect(this);
},
function(err, client, done){
if (err) throw err;
finish = done;
client.query(sql, this);
},
function(err, res){
// Release client to the pool
// should this be postponed to after the callback ?
// NOTE: if we pass a true value to finish() the client
// will be removed from the pool.
// We don't want this. Not now.
if ( finish ) finish();
callback(err, res)
}
);
};
// throw exception if illegal operations are detected
// NOTE: this check is weak hack, better database
// permissions should be used instead.
me.sanitize = function(sql, callback){
// NOTE: illegal table access is checked in main app
if (sql.match(/^\s+set\s+/i)){
var error = new SyntaxError("SET command is forbidden");
error.http_status = 403;
callback(error);
return;
}
callback(null,true);
};
return me;
};
/**
* Little hack for UI
* TODO: drop, fix in the UI (it's not documented in doc/API)
*
* @param {string} sql
* @param {number} limit
* @param {number} offset
* @returns {string} The wrapped SQL query with the limit window
*/
PSQL.window_sql = function(sql, limit, offset) {
// keeping it here for backwards compatibility
return new PSQLWrapper(sql).window(limit, offset).query();
};
module.exports = PSQL;

132
app/sql/psql_wrapper.js
View File

@ -1,132 +0,0 @@
'use strict';
var _ = require('underscore'),
util = require('util'),
SORT_ORDER_OPTIONS = {ASC: 1, DESC: 1},
REGEX_SELECT = /^\s*SELECT\s/i,
REGEX_INTO = /\sINTO\s+([^\s]+|"([^"]|"")*")\s*$/i;
function PSQLWrapper(sql) {
this.sqlQuery = sql.replace(/;\s*$/, '');
this.sqlClauses = {
orderBy: '',
limit: ''
};
}
/**
* Only window select functions (NOTE: "values" will be broken, "with" will be broken)
*
* @param {number} limit
* @param {number} offset
* @returns {PSQLWrapper}
*/
PSQLWrapper.prototype.window = function (limit, offset) {
if (!_.isNumber(limit) || !_.isNumber(offset)) {
return this;
}
this.sqlClauses.limit = util.format(' LIMIT %d OFFSET %d', limit, offset);
return this;
};
/**
*
* @param {string} column The name of the column to sort by
* @param {string} sortOrder Whether it's ASC or DESC ordering
* @returns {PSQLWrapper}
*/
PSQLWrapper.prototype.orderBy = function (column, sortOrder) {
if (!_.isString(column) || _.isEmpty(column)) {
return this;
}
this.sqlClauses.orderBy = util.format(' ORDER BY "%s"', column);
if (!_.isUndefined(sortOrder)) {
sortOrder = sortOrder.toUpperCase();
if (SORT_ORDER_OPTIONS[sortOrder]) {
this.sqlClauses.orderBy += util.format(' %s', sortOrder);
}
}
return this;
};
/**
* Builds an SQL query with extra clauses based on the builder calls.
*
* @returns {string} The SQL query with the extra clauses
*/
PSQLWrapper.prototype.query = function () {
if (_.isEmpty(this.sqlClauses.orderBy) && _.isEmpty(this.sqlClauses.limit)) {
return this.sqlQuery;
}
// Strip comments
this.sqlQuery = this.sqlQuery.replace(/(^|\n)\s*--.*\n/g, '');
var cte = '';
if (this.sqlQuery.match(/^\s*WITH\s/i)) {
var rem = this.sqlQuery, // analyzed portion of sql
q, // quote char
n = 0, // nested parens level
s = 0, // 0:outQuote, 1:inQuote
l;
while (1) {
l = rem.search(/[("')]/);
// console.log("REM Is " + rem);
if (l < 0) {
// console.log("Malformed SQL");
return this.sqlQuery;
}
var f = rem.charAt(l);
// console.log("n:" + n + " s:" + s + " l:" + l + " charAt(l):" + f + " charAt(l+1):" + rem.charAt(l+1));
if (s == 0) {
if (f == '(') {
++n;
} else if (f == ')') {
if (!--n) { // end of CTE
cte += rem.substr(0, l + 1);
rem = rem.substr(l + 1);
//console.log("Out of cte, rem is " + rem);
if (rem.search(/^s*,/) < 0) {
break;
} else {
continue; // cte and rem already updated
}
}
} else { // enter quoting
s = 1;
q = f;
}
} else if (f == q) {
if (rem.charAt(l + 1) == f) {
++l; // escaped
} else {
s = 0; // exit quoting
}
}
cte += rem.substr(0, l + 1);
rem = rem.substr(l + 1);
}
/*
console.log("cte: " + cte);
console.log("rem: " + rem);
*/
this.sqlQuery = rem; //sql.substr(l+1);
}
//console.log("SQL " + sql);
//console.log(" does " + ( sql.match(REGEX_SELECT) ? '' : 'not ' ) + "match REGEX_SELECT " + REGEX_SELECT);
//console.log(" does " + ( sql.match(REGEX_INTO) ? '' : 'not ' ) + "match REGEX_INTO " + REGEX_INTO);
if (this.sqlQuery.match(REGEX_SELECT) && !this.sqlQuery.match(REGEX_INTO)) {
return util.format(
'%sSELECT * FROM (%s) AS cdbq_1%s%s',
cte, this.sqlQuery, this.sqlClauses.orderBy, this.sqlClauses.limit
);
}
return cte + this.sqlQuery;
};
module.exports = PSQLWrapper;

3
package.json
View File

@ -20,6 +20,7 @@
"underscore.string": "~1.1.6",
"pg": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb1",
"express": "~2.5.11",
"cartodb-psql": "git://github.com/CartoDB/node-cartodb-psql.git#0.3.1",
"cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.8.0",
"step": "0.0.x",
"topojson": "0.0.8",
@ -37,7 +38,7 @@
"libxmljs": "~0.8.1"
},
"scripts": {
"test": "test/run_tests.sh ${RUNTESTFLAGS} test/unit/*.js test/unit/model/*.js test/unit/sql/*.js test/acceptance/*.js test/acceptance/export/*.js"
"test": "test/run_tests.sh ${RUNTESTFLAGS} test/unit/*.js test/unit/model/*.js test/acceptance/*.js test/acceptance/export/*.js"
},
"engines": { "node": ">= 0.4.1 < 0.9" }
}

112
test/unit/psql.test.js
View File

@ -1,112 +0,0 @@
require('../helper');
var _ = require('underscore')
, PSQL = require('../../app/models/psql')
, assert = require('assert');
var public_user = global.settings.db_pubuser;
var dbopts_auth = {
host: global.settings.db_host,
port: global.settings.db_port,
user: _.template(global.settings.db_user, {user_id: 1}),
dbname: _.template(global.settings.db_base_name, {user_id: 1}),
pass: _.template(global.settings.db_user_pass, {user_id: 1})
}
var dbopts_anon = _.clone(dbopts_auth);
dbopts_anon.user = global.settings.db_pubuser;
dbopts_anon.pass = global.settings.db_pubuser_pass;
suite('psql', function() {
test('test throws error if no args passed to constructor', function(){
var msg;
try{
var pg = new PSQL();
} catch (err){
msg = err.message;
}
assert.equal(msg, "Incorrect access parameters. If you are accessing via OAuth, please check your tokens are correct. For public users, please ensure your table is published.");
});
test('test private user can execute SELECTS on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "SELECT 1 as test_sum";
pg.query(sql, function(err, result){
assert.ok(!err, err);
assert.equal(result.rows[0].test_sum, 1);
done();
});
});
test('test private user can execute CREATE on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors; CREATE TABLE distributors (id integer, name varchar(40), UNIQUE(name))";
pg.query(sql, function(err, result){
assert.ok(_.isNull(err));
done();
});
});
test('test private user can execute INSERT on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors1; CREATE TABLE distributors1 (id integer, name varchar(40), UNIQUE(name))";
pg.query(sql, function(err, result){
sql = "INSERT INTO distributors1 (id, name) VALUES (1, 'fish')";
pg.query(sql,function(err, result){
assert.deepEqual(result.rows, []);
done();
});
});
});
test('test public user can execute SELECT on enabled tables', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors2; CREATE TABLE distributors2 (id integer, name varchar(40), UNIQUE(name)); GRANT SELECT ON distributors2 TO " + public_user + ";";
pg.query(sql, function(err, result){
pg = new PSQL(dbopts_anon)
pg.query("SELECT count(*) FROM distributors2", function(err, result){
assert.equal(result.rows[0].count, 0);
done();
});
});
});
test('test public user cannot execute INSERT on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors3; CREATE TABLE distributors3 (id integer, name varchar(40), UNIQUE(name)); GRANT SELECT ON distributors3 TO " + public_user + ";";
pg.query(sql, function(err, result){
pg = new PSQL(dbopts_anon);
pg.query("INSERT INTO distributors3 (id, name) VALUES (1, 'fishy')", function(err, result){
assert.equal(err.message, 'permission denied for relation distributors3');
done();
});
});
});
test('dbkey depends on dbopts', function(){
var opt1 = _.clone(dbopts_anon);
opt1.dbname = 'dbname1';
var pg1 = new PSQL(opt1);
var opt2 = _.clone(dbopts_anon);
opt2.dbname = 'dbname2';
var pg2 = new PSQL(opt2);
assert.ok(pg1.dbkey() !== pg2.dbkey(),
'both PSQL object using same dbkey ' + pg1.dbkey());
assert.ok(_.isString(pg1.dbkey()), "pg1 dbkey is " + pg1.dbkey());
});
test('eventedQuery provisions a cancel mechanism to abort queries', function (done) {
var psql = new PSQL(dbopts_auth);
psql.eventedQuery("SELECT 1 as foo", function(err, query, queryCanceller) {
assert.ok(_.isFunction(queryCanceller));
done();
});
});
});

115
test/unit/sql/psql_wrapper.test.js
View File

@ -1,115 +0,0 @@
var _ = require('underscore'),
PSQLWrapper = require('../../../app/sql/psql_wrapper'),
assert = require('assert');
// NOTE: intentionally mixed-case and space-padded
var simpleSql = "\n \tSEleCT * from table1";
suite('psql_wrapper', function() {
test('Windowed SQL with simple select', function(){
var out = new PSQLWrapper(simpleSql).window(1, 0).query();
assert.equal(out, "SELECT * FROM (" + simpleSql + ") AS cdbq_1 LIMIT 1 OFFSET 0");
});
test('Windowed SQL with CTE select', function(){
// NOTE: intentionally mixed-case and space-padded
var cte = "\n \twiTh x as( update test set x=x+1)";
var select = "\n \tSEleCT * from x";
var sql = cte + select;
var out = new PSQLWrapper(sql).window(1, 0).query();
assert.equal(out, cte + "SELECT * FROM (" + select + ") AS cdbq_1 LIMIT 1 OFFSET 0");
});
test('Windowed SQL with CTE update', function(){
// NOTE: intentionally mixed-case and space-padded
var cte = "\n \twiTh a as( update test set x=x+1)";
var upd = "\n \tupdate tost set y=x from x";
var sql = cte + upd;
var out = new PSQLWrapper(sql).window(1, 0).query();
assert.equal(out, sql);
});
test('Windowed SQL with complex CTE and insane quoting', function(){
// NOTE: intentionally mixed-case and space-padded
var cte = "\n \twiTh \"('a\" as( update \"\"\"test)\" set x='x'+1), \")b(\" as ( select ')))\"' from z )";
var sel = "\n \tselect '\"' from x";
var sql = cte + sel;
var out = new PSQLWrapper(sql).window(1, 0).query();
assert.equal(out, cte + "SELECT * FROM (" + sel + ") AS cdbq_1 LIMIT 1 OFFSET 0");
});
test('Different instances return different queries', function() {
var aWrapper = new PSQLWrapper('select 1');
var bWrapper = new PSQLWrapper('select * from databaseB');
assert.notEqual(aWrapper, bWrapper);
assert.notEqual(aWrapper.query(), bWrapper.query(), 'queries should be different');
});
test('Order by SQL with simple select and empty column name returns original query', function() {
var expectedSql = simpleSql;
var outputSql = new PSQLWrapper(simpleSql).orderBy('').query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and no sort order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo"';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo').query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and invalid sort order use no sort order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo"';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo', "BAD_SORT_ORDER").query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and asc order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo" ASC';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo', "asc").query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and DESC order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo" DESC';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo', "DESC").query();
assert.equal(outputSql, expectedSql);
});
test('Query with ending semicolon returns without it', function() {
var expectedSql = 'select a, ( a - min(a) over() ) / ( ( max(a) over () - min(a) over () ) / 4 ) as interval from ( select test as a from quantile_test ) as f',
query = expectedSql + ';';
var outputSql = new PSQLWrapper(query).query();
assert.equal(outputSql, expectedSql);
});
test('Several queries with semicolon get only last semicolon removed', function() {
var expectedSql = 'SELECT 1; SELECT 2; SELECT 3',
query = expectedSql + ';';
var outputSql = new PSQLWrapper(query).query();
assert.equal(outputSql, expectedSql);
});
});