Merge branch 'master' into upgrade-dependencies

Conflicts:
	app/models/psql.js
	npm-shrinkwrap.json
	package.json
This commit is contained in:
Raul Ochoa 2014-08-14 14:19:20 +02:00
commit 0aa1b0e2e9
10 changed files with 413 additions and 1071 deletions

View File

@ -1,6 +1,11 @@
1.15.0 - 2014-mm-dd 1.15.0 - 2014-mm-dd
------------------- -------------------
Enhancements:
* Upgrades cartodb-redis
1.14.1 - 2014-08-08 1.14.1 - 2014-08-08
------------------- -------------------

View File

@ -33,8 +33,7 @@ var express = require('express')
, Profiler = require('step-profiler') , Profiler = require('step-profiler')
, StatsD = require('node-statsd').StatsD , StatsD = require('node-statsd').StatsD
, MetadataDB = require('cartodb-redis') , MetadataDB = require('cartodb-redis')
, PSQL = require(global.settings.app_root + '/app/models/psql') , PSQL = require('cartodb-psql')
, PSQLWrapper = require(global.settings.app_root + '/app/sql/psql_wrapper')
, CdbRequest = require(global.settings.app_root + '/app/models/cartodb_request') , CdbRequest = require(global.settings.app_root + '/app/models/cartodb_request')
, AuthApi = require(global.settings.app_root + '/app/auth/auth_api') , AuthApi = require(global.settings.app_root + '/app/auth/auth_api')
, _ = require('underscore') , _ = require('underscore')
@ -351,7 +350,7 @@ function handleQuery(req, res) {
return false; return false;
} else { } else {
//TODO: sanitize cdbuser //TODO: sanitize cdbuser
console.log("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$"); console.log("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)");
pg.query("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)", function (err, result) { pg.query("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)", function (err, result) {
if (err) { if (err) {
self(err); self(err);
@ -446,7 +445,7 @@ function handleQuery(req, res) {
checkAborted('generateFormat'); checkAborted('generateFormat');
// TODO: drop this, fix UI! // TODO: drop this, fix UI!
sql = new PSQLWrapper(sql).orderBy(orderBy, sortOrder).window(limit, offset).query(); sql = new PSQL.QueryWrapper(sql).orderBy(orderBy, sortOrder).window(limit, offset).query();
var opts = { var opts = {
dbopts: dbopts, dbopts: dbopts,

View File

@ -2,7 +2,7 @@ var crypto = require('crypto'),
Step = require('step'), Step = require('step'),
fs = require('fs'), fs = require('fs'),
_ = require('underscore'), _ = require('underscore'),
PSQL = require(global.settings.app_root + '/app/models/psql'), PSQL = require('cartodb-psql'),
spawn = require('child_process').spawn; spawn = require('child_process').spawn;
// Keeps track of what's waiting baking for export // Keeps track of what's waiting baking for export

View File

@ -1,5 +1,5 @@
var Step = require('step'), var Step = require('step'),
PSQL = require(global.settings.app_root + '/app/models/psql'); PSQL = require('cartodb-psql');
function PostgresFormat(id) { function PostgresFormat(id) {
this.id = id; this.id = id;

View File

@ -1,284 +0,0 @@
var _ = require('underscore'),
PSQLWrapper = require('../sql/psql_wrapper'),
Step = require('step'),
pg = require('pg');//.native; // disabled for now due to: https://github.com/brianc/node-postgres/issues/48
_.mixin(require('underscore.string'));
// Max database connections in the pool
// Subsequent connections will block waiting for a free slot
pg.defaults.poolSize = global.settings.db_pool_size || 16;
// Milliseconds of idle time before removing connection from pool
pg.defaults.poolIdleTimeout = global.settings.db_pool_idleTimeout || 30000;
// Frequency to check for idle clients within the pool, ms
pg.defaults.reapIntervalMillis = global.settings.db_pool_reapInterval || 1000;
pg.on('error', function(err, client) {
console.log("PostgreSQL connection error: " + err);
});
// Workaround for https://github.com/Vizzuality/CartoDB-SQL-API/issues/100
var types = pg.types;
var arrayParser = require('pg-types').arrayParser;
var floatParser = function(val) {
return parseFloat(val);
};
var floatArrayParser = function(val) {
if(!val) { return null; }
var p = arrayParser.create(val, function(entry) {
return floatParser(entry);
});
return p.parse();
};
types.setTypeParser(20, floatParser); // int8
types.setTypeParser(700, floatParser); // float4
types.setTypeParser(701, floatParser); // float8
types.setTypeParser(1700, floatParser); // numeric
types.setTypeParser(1021, floatArrayParser); // _float4
types.setTypeParser(1022, floatArrayParser); // _float8
types.setTypeParser(1231, floatArrayParser); // _numeric
types.setTypeParser(1016, floatArrayParser); // _int8
// Standard type->name mappnig (up to oid=2000)
var stdTypeName = {
16: 'bool',
17: 'bytea',
20: 'int8',
21: 'int2',
23: 'int4',
25: 'text',
26: 'oid',
114: 'JSON',
700: 'float4',
701: 'float8',
1000: '_bool',
1015: '_varchar',
1042: 'bpchar',
1043: 'varchar',
1005: '_int2',
1007: '_int4',
1014: '_bpchar',
1016: '_int8',
1021: '_float4',
1022: '_float8',
1008: '_regproc',
1009: '_text',
1082: 'date',
1114: 'timestamp',
1182: '_date',
1184: 'timestampz',
1186: 'interval',
1231: '_numeric',
1700: 'numeric'
};
// Holds a typeId->typeName mapping for each
// database ever connected to
var extTypeName = {};
// PSQL
//
// A simple postgres wrapper with logic about username and database to connect
//
// * intended for use with pg_bouncer
// * defaults to connecting with a "READ ONLY" user to given DB if not passed a specific user_id
//
// @param opts connection options:
// user: database username
// pass: database user password
// host: database host
// port: database port
// dbname: database name
//
var PSQL = function(dbopts) {
var error_text = "Incorrect access parameters. If you are accessing via OAuth, please check your tokens are correct. For public users, please ensure your table is published."
if ( ! dbopts || ( !_.isString(dbopts.user) && !_.isString(dbopts.dbname)))
{
// console.log("DBOPTS: "); console.dir(dbopts);
throw new Error(error_text);
}
var me = {
dbopts: dbopts
};
me.username = function(){
return this.dbopts.user;
};
me.password = function(){
return this.dbopts.pass;
};
me.database = function(){
return this.dbopts.dbname;
};
me.dbhost = function(){
return this.dbopts.host;
};
me.dbport = function(){
return this.dbopts.port;
};
me.conString = "tcp://" + me.username() +
":" + me.password() + // this line only if not-null ?
"@" +
me.dbhost() + ":" +
me.dbport() + "/" +
me.database();
me.dbkey = function(){
return this.database(); // + ":" + this.dbhost() + ":" + me.dbport();
};
me.ensureTypeCache = function(cb) {
var db = this.dbkey();
if ( extTypeName[db] ) { cb(); return; }
pg.connect(this.conString, function(err, client, done) {
if ( err ) { cb(err); return; }
var types = ["'geometry'","'raster'"]; // types of interest
client.query("SELECT oid, typname FROM pg_type where typname in (" + types.join(',') + ")", function(err,res) {
done();
if ( err ) { cb(err); return; }
var cache = {};
res.rows.map(function(r) {
cache[r.oid] = r.typname;
});
extTypeName[db] = cache;
cb();
});
});
}
// Return type name for a type identifier
//
// Possibly returns undefined, for unkonwn (uncached)
//
me.typeName = function(typeId) {
return stdTypeName[typeId] ? stdTypeName[typeId] : extTypeName[this.dbkey()][typeId];
}
me.connect = function(cb){
var that = this;
this.ensureTypeCache(function(err) {
if ( err ) cb(err);
else pg.connect(that.conString, cb);
});
};
me.eventedQuery = function(sql, callback){
var that = this;
Step(
function(){
that.sanitize(sql, this);
},
function(err, clean){
if (err) throw err;
that.connect(this);
},
function(err, client, done){
var next = this;
if (err) throw err;
var query = client.query(sql);
// forward notices to query
var noticeListener = function() {
query.emit('notice', arguments);
};
client.on('notice', noticeListener);
// NOTE: for some obscure reason passing "done" directly
// as the listener works but can be slower
// (by x2 factor!)
query.on('end', function() {
client.removeListener('notice', noticeListener);
done();
});
next(null, query, client);
},
function(err, query, client){
var queryCanceller = function() {
pg.cancel(undefined, client, query);
};
callback(err, query, queryCanceller);
}
);
};
me.quoteIdentifier = function(str) {
return pg.Client.prototype.escapeIdentifier(str);
};
me.escapeLiteral = function(s) {
return pg.Client.prototype.escapeLiteral(str);
};
me.query = function(sql, callback){
var that = this;
var finish;
Step(
function(){
that.sanitize(sql, this);
},
function(err, clean){
if (err) throw err;
that.connect(this);
},
function(err, client, done){
if (err) throw err;
finish = done;
client.query(sql, this);
},
function(err, res){
// Release client to the pool
// should this be postponed to after the callback ?
// NOTE: if we pass a true value to finish() the client
// will be removed from the pool.
// We don't want this. Not now.
if ( finish ) finish();
callback(err, res)
}
);
};
// throw exception if illegal operations are detected
// NOTE: this check is weak hack, better database
// permissions should be used instead.
me.sanitize = function(sql, callback){
// NOTE: illegal table access is checked in main app
if (sql.match(/^\s+set\s+/i)){
var error = new SyntaxError("SET command is forbidden");
error.http_status = 403;
callback(error);
return;
}
callback(null,true);
};
return me;
};
/**
* Little hack for UI
* TODO: drop, fix in the UI (it's not documented in doc/API)
*
* @param {string} sql
* @param {number} limit
* @param {number} offset
* @returns {string} The wrapped SQL query with the limit window
*/
PSQL.window_sql = function(sql, limit, offset) {
// keeping it here for backwards compatibility
return new PSQLWrapper(sql).window(limit, offset).query();
};
module.exports = PSQL;

View File

@ -1,132 +0,0 @@
'use strict';
var _ = require('underscore'),
util = require('util'),
SORT_ORDER_OPTIONS = {ASC: 1, DESC: 1},
REGEX_SELECT = /^\s*SELECT\s/i,
REGEX_INTO = /\sINTO\s+([^\s]+|"([^"]|"")*")\s*$/i;
function PSQLWrapper(sql) {
this.sqlQuery = sql.replace(/;\s*$/, '');
this.sqlClauses = {
orderBy: '',
limit: ''
};
}
/**
* Only window select functions (NOTE: "values" will be broken, "with" will be broken)
*
* @param {number} limit
* @param {number} offset
* @returns {PSQLWrapper}
*/
PSQLWrapper.prototype.window = function (limit, offset) {
if (!_.isNumber(limit) || !_.isNumber(offset)) {
return this;
}
this.sqlClauses.limit = util.format(' LIMIT %d OFFSET %d', limit, offset);
return this;
};
/**
*
* @param {string} column The name of the column to sort by
* @param {string} sortOrder Whether it's ASC or DESC ordering
* @returns {PSQLWrapper}
*/
PSQLWrapper.prototype.orderBy = function (column, sortOrder) {
if (!_.isString(column) || _.isEmpty(column)) {
return this;
}
this.sqlClauses.orderBy = util.format(' ORDER BY "%s"', column);
if (!_.isUndefined(sortOrder)) {
sortOrder = sortOrder.toUpperCase();
if (SORT_ORDER_OPTIONS[sortOrder]) {
this.sqlClauses.orderBy += util.format(' %s', sortOrder);
}
}
return this;
};
/**
* Builds an SQL query with extra clauses based on the builder calls.
*
* @returns {string} The SQL query with the extra clauses
*/
PSQLWrapper.prototype.query = function () {
if (_.isEmpty(this.sqlClauses.orderBy) && _.isEmpty(this.sqlClauses.limit)) {
return this.sqlQuery;
}
// Strip comments
this.sqlQuery = this.sqlQuery.replace(/(^|\n)\s*--.*\n/g, '');
var cte = '';
if (this.sqlQuery.match(/^\s*WITH\s/i)) {
var rem = this.sqlQuery, // analyzed portion of sql
q, // quote char
n = 0, // nested parens level
s = 0, // 0:outQuote, 1:inQuote
l;
while (1) {
l = rem.search(/[("')]/);
// console.log("REM Is " + rem);
if (l < 0) {
// console.log("Malformed SQL");
return this.sqlQuery;
}
var f = rem.charAt(l);
// console.log("n:" + n + " s:" + s + " l:" + l + " charAt(l):" + f + " charAt(l+1):" + rem.charAt(l+1));
if (s == 0) {
if (f == '(') {
++n;
} else if (f == ')') {
if (!--n) { // end of CTE
cte += rem.substr(0, l + 1);
rem = rem.substr(l + 1);
//console.log("Out of cte, rem is " + rem);
if (rem.search(/^s*,/) < 0) {
break;
} else {
continue; // cte and rem already updated
}
}
} else { // enter quoting
s = 1;
q = f;
}
} else if (f == q) {
if (rem.charAt(l + 1) == f) {
++l; // escaped
} else {
s = 0; // exit quoting
}
}
cte += rem.substr(0, l + 1);
rem = rem.substr(l + 1);
}
/*
console.log("cte: " + cte);
console.log("rem: " + rem);
*/
this.sqlQuery = rem; //sql.substr(l+1);
}
//console.log("SQL " + sql);
//console.log(" does " + ( sql.match(REGEX_SELECT) ? '' : 'not ' ) + "match REGEX_SELECT " + REGEX_SELECT);
//console.log(" does " + ( sql.match(REGEX_INTO) ? '' : 'not ' ) + "match REGEX_INTO " + REGEX_INTO);
if (this.sqlQuery.match(REGEX_SELECT) && !this.sqlQuery.match(REGEX_INTO)) {
return util.format(
'%sSELECT * FROM (%s) AS cdbq_1%s%s',
cte, this.sqlQuery, this.sqlClauses.orderBy, this.sqlClauses.limit
);
}
return cte + this.sqlQuery;
};
module.exports = PSQLWrapper;

814
npm-shrinkwrap.json generated
View File

@ -3,47 +3,30 @@
"version": "1.15.0", "version": "1.15.0",
"dependencies": { "dependencies": {
"cartodb-redis": { "cartodb-redis": {
"version": "0.8.0", "version": "0.9.0",
"from": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.8.0", "from": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0",
"resolved": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.8.0",
"dependencies": { "dependencies": {
"underscore": {
"version": "1.3.3",
"from": "underscore@~1.3",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.3.3.tgz"
},
"strftime": { "strftime": {
"version": "0.6.2", "version": "0.8.2"
"from": "strftime@~0.6.0",
"resolved": "https://registry.npmjs.org/strftime/-/strftime-0.6.2.tgz"
}, },
"redis-mpool": { "redis-mpool": {
"version": "0.0.4", "version": "0.1.0",
"from": "redis-mpool@git://github.com/CartoDB/node-redis-mpool.git#0.0.4", "from": "git://github.com/CartoDB/node-redis-mpool.git#0.1.0",
"resolved": "git://github.com/CartoDB/node-redis-mpool.git#0ba3e2929f9a4ccc964fc2ca6fbfb0f278b74ab8",
"dependencies": { "dependencies": {
"generic-pool": { "generic-pool": {
"version": "2.0.4", "version": "2.1.1"
"from": "generic-pool@~2.0.3",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.0.4.tgz"
}, },
"redis": { "redis": {
"version": "0.8.6", "version": "0.12.1"
"from": "redis@~0.8.3"
}, },
"hiredis": { "hiredis": {
"version": "0.1.17", "version": "0.1.17",
"from": "hiredis@~0.1.14",
"resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.1.17.tgz",
"dependencies": { "dependencies": {
"bindings": { "bindings": {
"version": "1.2.1", "version": "1.2.1"
"from": "bindings@*"
}, },
"nan": { "nan": {
"version": "1.1.2", "version": "1.1.2"
"from": "nan@~1.1.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-1.1.2.tgz"
} }
} }
} }
@ -51,427 +34,426 @@
} }
} }
}, },
"cartodb-psql": {
"version": "0.3.1",
"from": "git://github.com/CartoDB/node-cartodb-psql.git#0.3.1",
"dependencies": {
"pg": {
"version": "2.6.2-cdb1",
"from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb1",
"dependencies": {
"generic-pool": {
"version": "2.0.3"
},
"buffer-writer": {
"version": "1.0.0"
}
}
}
}
},
"step-profiler": {
"version": "0.1.0",
"from": "https://github.com/CartoDB/node-step-profiler/tarball/0.1.0"
},
"underscore": {
"version": "1.6.0"
},
"underscore.string": {
"version": "2.3.3"
},
"express": { "express": {
"version": "2.5.11", "version": "2.5.11",
"from": "express@~2.5.11",
"resolved": "https://registry.npmjs.org/express/-/express-2.5.11.tgz",
"dependencies": { "dependencies": {
"connect": { "connect": {
"version": "1.9.2", "version": "1.9.2",
"from": "connect@1.x",
"resolved": "https://registry.npmjs.org/connect/-/connect-1.9.2.tgz",
"dependencies": { "dependencies": {
"formidable": { "formidable": {
"version": "1.0.15", "version": "1.0.15"
"from": "formidable@1.0.x",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.15.tgz"
} }
} }
}, },
"mime": { "mime": {
"version": "1.2.4", "version": "1.2.4"
"from": "mime@1.2.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.2.4.tgz"
}, },
"qs": { "qs": {
"version": "0.4.2", "version": "0.4.2"
"from": "qs@0.4.x",
"resolved": "https://registry.npmjs.org/qs/-/qs-0.4.2.tgz"
}, },
"mkdirp": { "mkdirp": {
"version": "0.3.0", "version": "0.3.0"
"from": "mkdirp@0.3.0"
}
}
},
"log4js": {
"version": "0.6.16",
"from": "log4js@~0.6.16",
"dependencies": {
"async": {
"version": "0.1.15",
"from": "async@0.1.15"
},
"semver": {
"version": "1.1.4",
"from": "semver@~1.1.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-1.1.4.tgz"
},
"readable-stream": {
"version": "1.0.27-1",
"from": "readable-stream@~1.0.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
"dependencies": {
"core-util-is": {
"version": "1.0.1",
"from": "core-util-is@~1.0.0",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz"
},
"isarray": {
"version": "0.0.1",
"from": "isarray@0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
},
"string_decoder": {
"version": "0.10.25-1",
"from": "string_decoder@~0.10.x",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.25-1.tgz"
},
"inherits": {
"version": "2.0.1",
"from": "inherits@~2.0.1",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
}
}
}
}
},
"lru-cache": {
"version": "2.5.0",
"from": "lru-cache@~2.5.0"
},
"node-statsd": {
"version": "0.0.7",
"from": "node-statsd@~0.0.7",
"resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.0.7.tgz"
},
"oauth-client": {
"version": "0.3.0",
"from": "oauth-client@0.3.0",
"resolved": "https://registry.npmjs.org/oauth-client/-/oauth-client-0.3.0.tgz",
"dependencies": {
"node-uuid": {
"version": "1.1.0",
"from": "node-uuid@1.1.0"
}
}
},
"pg": {
"version": "3.4.1",
"from": "pg@~3.4.1",
"resolved": "https://registry.npmjs.org/pg/-/pg-3.4.1.tgz",
"dependencies": {
"generic-pool": {
"version": "2.1.1",
"from": "generic-pool@2.1.1",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz"
},
"buffer-writer": {
"version": "1.0.0",
"from": "buffer-writer@1.0.0",
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz"
},
"pgpass": {
"version": "0.0.3",
"from": "pgpass@0.0.3",
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-0.0.3.tgz",
"dependencies": {
"split": {
"version": "0.3.0",
"from": "split@~0.3",
"dependencies": {
"through": {
"version": "2.3.4",
"from": "through@2"
}
}
}
}
},
"nan": {
"version": "1.2.0",
"from": "nan@~1.2.0"
},
"packet-reader": {
"version": "0.2.0",
"from": "packet-reader@0.2.0",
"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.2.0.tgz"
},
"pg-connection-string": {
"version": "0.1.1",
"from": "pg-connection-string@0.1.1",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.1.tgz"
}
}
},
"pg-types": {
"version": "1.4.0",
"from": "pg-types@1.4.0",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.4.0.tgz"
},
"rollbar": {
"version": "0.3.12",
"from": "rollbar@~0.3.12",
"dependencies": {
"node-uuid": {
"version": "1.4.1",
"from": "node-uuid@1.4.x",
"resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.1.tgz"
},
"lru-cache": {
"version": "2.2.4",
"from": "lru-cache@~2.2.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.2.4.tgz"
},
"json-stringify-safe": {
"version": "5.0.0",
"from": "json-stringify-safe@~5.0.0",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.0.tgz"
} }
} }
}, },
"step": { "step": {
"version": "0.0.5", "version": "0.0.5"
"from": "step@~0.0.5",
"resolved": "https://registry.npmjs.org/step/-/step-0.0.5.tgz"
},
"step-profiler": {
"version": "0.1.0",
"from": "https://github.com/CartoDB/node-step-profiler/tarball/0.1.0",
"resolved": "https://github.com/CartoDB/node-step-profiler/tarball/0.1.0"
}, },
"topojson": { "topojson": {
"version": "1.6.15", "version": "0.0.8",
"from": "topojson@1.6.15",
"resolved": "https://registry.npmjs.org/topojson/-/topojson-1.6.15.tgz",
"dependencies": { "dependencies": {
"d3": {
"version": "3.4.11",
"from": "d3@3",
"resolved": "https://registry.npmjs.org/d3/-/d3-3.4.11.tgz",
"dependencies": {
"jsdom": {
"version": "0.5.7",
"from": "jsdom@0.5.7",
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-0.5.7.tgz",
"dependencies": {
"htmlparser": {
"version": "1.7.7",
"from": "htmlparser@1.x",
"resolved": "https://registry.npmjs.org/htmlparser/-/htmlparser-1.7.7.tgz"
},
"nwmatcher": {
"version": "1.3.3",
"from": "nwmatcher@~1.3.1",
"resolved": "https://registry.npmjs.org/nwmatcher/-/nwmatcher-1.3.3.tgz"
},
"request": {
"version": "2.40.0",
"from": "request@2.x",
"dependencies": {
"qs": {
"version": "1.0.2",
"from": "qs@~1.0.0"
},
"json-stringify-safe": {
"version": "5.0.0",
"from": "json-stringify-safe@~5.0.0",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.0.tgz"
},
"mime-types": {
"version": "1.0.2",
"from": "mime-types@~1.0.1"
},
"forever-agent": {
"version": "0.5.2",
"from": "forever-agent@~0.5.0",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.5.2.tgz"
},
"node-uuid": {
"version": "1.4.1",
"from": "node-uuid@~1.4.0",
"resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.1.tgz"
},
"tough-cookie": {
"version": "0.12.1",
"from": "tough-cookie@>=0.12.0",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-0.12.1.tgz",
"dependencies": {
"punycode": {
"version": "1.3.1",
"from": "punycode@>=0.2.0"
}
}
},
"form-data": {
"version": "0.1.4",
"from": "form-data@~0.1.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-0.1.4.tgz",
"dependencies": {
"combined-stream": {
"version": "0.0.5",
"from": "combined-stream@~0.0.4",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.5.tgz",
"dependencies": {
"delayed-stream": {
"version": "0.0.5",
"from": "delayed-stream@0.0.5",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
}
}
},
"mime": {
"version": "1.2.11",
"from": "mime@~1.2.11"
},
"async": {
"version": "0.9.0",
"from": "async@~0.9.0"
}
}
},
"tunnel-agent": {
"version": "0.4.0",
"from": "tunnel-agent@~0.4.0"
},
"http-signature": {
"version": "0.10.0",
"from": "http-signature@~0.10.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.10.0.tgz",
"dependencies": {
"assert-plus": {
"version": "0.1.2",
"from": "assert-plus@0.1.2",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.2.tgz"
},
"asn1": {
"version": "0.1.11",
"from": "asn1@0.1.11",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.1.11.tgz"
},
"ctype": {
"version": "0.5.2",
"from": "ctype@0.5.2",
"resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.2.tgz"
}
}
},
"oauth-sign": {
"version": "0.3.0",
"from": "oauth-sign@~0.3.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.3.0.tgz"
},
"hawk": {
"version": "1.1.1",
"from": "hawk@1.1.1",
"dependencies": {
"hoek": {
"version": "0.9.1",
"from": "hoek@0.9.x",
"resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz"
},
"boom": {
"version": "0.4.2",
"from": "boom@0.4.x",
"resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz"
},
"cryptiles": {
"version": "0.2.2",
"from": "cryptiles@0.2.x",
"resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.2.2.tgz"
},
"sntp": {
"version": "0.2.4",
"from": "sntp@0.2.x",
"resolved": "https://registry.npmjs.org/sntp/-/sntp-0.2.4.tgz"
}
}
},
"aws-sign2": {
"version": "0.5.0",
"from": "aws-sign2@~0.5.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz"
},
"stringstream": {
"version": "0.0.4",
"from": "stringstream@~0.0.4"
}
}
},
"cssom": {
"version": "0.2.5",
"from": "cssom@~0.2.5",
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.2.5.tgz"
},
"cssstyle": {
"version": "0.2.14",
"from": "cssstyle@~0.2.3",
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-0.2.14.tgz",
"dependencies": {
"cssom": {
"version": "0.3.0",
"from": "cssom@0.3.x",
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.0.tgz"
}
}
},
"contextify": {
"version": "0.1.8",
"from": "contextify@~0.1.5",
"resolved": "https://registry.npmjs.org/contextify/-/contextify-0.1.8.tgz",
"dependencies": {
"bindings": {
"version": "1.2.1",
"from": "bindings@*"
},
"nan": {
"version": "1.0.0",
"from": "nan@~1.0.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-1.0.0.tgz"
}
}
}
}
}
}
},
"d3-geo-projection": {
"version": "0.2.10",
"from": "d3-geo-projection@0.2",
"resolved": "https://registry.npmjs.org/d3-geo-projection/-/d3-geo-projection-0.2.10.tgz"
},
"optimist": { "optimist": {
"version": "0.3.7", "version": "0.3.5",
"from": "optimist@0.3",
"resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.7.tgz",
"dependencies": { "dependencies": {
"wordwrap": { "wordwrap": {
"version": "0.0.2", "version": "0.0.2"
"from": "wordwrap@~0.0.2",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz"
}
}
},
"queue-async": {
"version": "1.0.7",
"from": "queue-async@1.0",
"resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz"
},
"rw": {
"version": "0.1.4",
"from": "rw@0.1",
"resolved": "https://registry.npmjs.org/rw/-/rw-0.1.4.tgz"
},
"shapefile": {
"version": "0.3.0",
"from": "shapefile@0.3",
"resolved": "https://registry.npmjs.org/shapefile/-/shapefile-0.3.0.tgz",
"dependencies": {
"iconv-lite": {
"version": "0.2.11",
"from": "iconv-lite@0.2"
} }
} }
} }
} }
}, },
"underscore": { "oauth-client": {
"version": "1.6.0", "version": "0.3.0",
"from": "underscore@~1.6.0" "dependencies": {
"node-uuid": {
"version": "1.1.0"
}
}
}, },
"underscore.string": { "lru-cache": {
"version": "2.3.3", "version": "2.5.0"
"from": "underscore.string@~2.3.3" },
"log4js": {
"version": "0.6.16",
"dependencies": {
"async": {
"version": "0.1.15"
},
"semver": {
"version": "1.1.4"
},
"readable-stream": {
"version": "1.0.27-1",
"dependencies": {
"core-util-is": {
"version": "1.0.1"
},
"isarray": {
"version": "0.0.1"
},
"string_decoder": {
"version": "0.10.25-1"
},
"inherits": {
"version": "2.0.1"
}
}
}
}
},
"rollbar": {
"version": "0.3.12",
"dependencies": {
"node-uuid": {
"version": "1.4.1"
},
"lru-cache": {
"version": "2.2.4"
},
"json-stringify-safe": {
"version": "5.0.0"
}
}
},
"node-statsd": {
"version": "0.0.7"
},
"redis": {
"version": "0.7.1"
},
"libxmljs": {
"version": "0.8.1",
"dependencies": {
"bindings": {
"version": "1.0.0"
}
}
},
"mocha": {
"version": "1.14.0",
"dependencies": {
"commander": {
"version": "2.0.0"
},
"growl": {
"version": "1.7.0"
},
"jade": {
"version": "0.26.3",
"dependencies": {
"commander": {
"version": "0.6.1"
},
"mkdirp": {
"version": "0.3.0"
}
}
},
"diff": {
"version": "1.0.7"
},
"debug": {
"version": "1.0.4",
"dependencies": {
"ms": {
"version": "0.6.2"
}
}
},
"mkdirp": {
"version": "0.3.5"
},
"glob": {
"version": "3.2.3",
"dependencies": {
"minimatch": {
"version": "0.2.14",
"dependencies": {
"sigmund": {
"version": "1.0.0"
}
}
},
"graceful-fs": {
"version": "2.0.3"
},
"inherits": {
"version": "2.0.1"
}
}
}
}
},
"zipfile": {
"version": "0.5.3",
"dependencies": {
"nan": {
"version": "1.2.0"
},
"node-pre-gyp": {
"version": "0.5.19",
"dependencies": {
"nopt": {
"version": "2.2.1",
"dependencies": {
"abbrev": {
"version": "1.0.5"
}
}
},
"npmlog": {
"version": "0.0.6",
"dependencies": {
"ansi": {
"version": "0.2.1"
}
}
},
"request": {
"version": "2.36.0",
"dependencies": {
"qs": {
"version": "0.6.6"
},
"json-stringify-safe": {
"version": "5.0.0"
},
"mime": {
"version": "1.2.11"
},
"forever-agent": {
"version": "0.5.2"
},
"node-uuid": {
"version": "1.4.1"
},
"tough-cookie": {
"version": "0.12.1",
"dependencies": {
"punycode": {
"version": "1.2.4"
}
}
},
"form-data": {
"version": "0.1.2",
"dependencies": {
"combined-stream": {
"version": "0.0.4",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
}
}
},
"async": {
"version": "0.2.10"
}
}
},
"tunnel-agent": {
"version": "0.4.0"
},
"http-signature": {
"version": "0.10.0",
"dependencies": {
"assert-plus": {
"version": "0.1.2"
},
"asn1": {
"version": "0.1.11"
},
"ctype": {
"version": "0.5.2"
}
}
},
"oauth-sign": {
"version": "0.3.0"
},
"hawk": {
"version": "1.0.0",
"dependencies": {
"hoek": {
"version": "0.9.1"
},
"boom": {
"version": "0.4.2"
},
"cryptiles": {
"version": "0.2.2"
},
"sntp": {
"version": "0.2.4"
}
}
},
"aws-sign2": {
"version": "0.5.0"
}
}
},
"semver": {
"version": "2.3.0"
},
"tar": {
"version": "0.1.19",
"dependencies": {
"inherits": {
"version": "2.0.1"
},
"block-stream": {
"version": "0.0.7"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"mkdirp": {
"version": "0.3.5"
},
"graceful-fs": {
"version": "2.0.3"
}
}
}
}
},
"tar-pack": {
"version": "2.0.0",
"dependencies": {
"uid-number": {
"version": "0.0.3"
},
"once": {
"version": "1.1.1"
},
"debug": {
"version": "0.7.4"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"mkdirp": {
"version": "0.3.5"
},
"graceful-fs": {
"version": "2.0.3"
},
"inherits": {
"version": "2.0.1"
}
}
},
"fstream-ignore": {
"version": "0.0.7",
"dependencies": {
"minimatch": {
"version": "0.2.14",
"dependencies": {
"lru-cache": {
"version": "2.5.0"
},
"sigmund": {
"version": "1.0.0"
}
}
},
"inherits": {
"version": "2.0.1"
}
}
},
"readable-stream": {
"version": "1.0.27-1",
"dependencies": {
"core-util-is": {
"version": "1.0.1"
},
"isarray": {
"version": "0.0.1"
},
"string_decoder": {
"version": "0.10.25-1"
},
"inherits": {
"version": "2.0.1"
}
}
},
"graceful-fs": {
"version": "1.2.3"
}
}
},
"mkdirp": {
"version": "0.5.0",
"dependencies": {
"minimist": {
"version": "0.0.8"
}
}
},
"rc": {
"version": "0.4.0",
"dependencies": {
"minimist": {
"version": "0.0.10"
},
"deep-extend": {
"version": "0.2.10"
},
"strip-json-comments": {
"version": "0.1.3"
},
"ini": {
"version": "1.1.0"
}
}
},
"rimraf": {
"version": "2.2.8"
}
}
}
}
} }
} }
} }

View File

@ -16,19 +16,18 @@
"Sandro Santilli <strk@vizzuality.com>" "Sandro Santilli <strk@vizzuality.com>"
], ],
"dependencies": { "dependencies": {
"cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.8.0", "cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0",
"cartodb-psql": "git://github.com/CartoDB/node-cartodb-psql.git#0.3.1",
"step-profiler": "https://github.com/CartoDB/node-step-profiler/tarball/0.1.0", "step-profiler": "https://github.com/CartoDB/node-step-profiler/tarball/0.1.0",
"underscore" : "~1.6.0", "underscore" : "~1.6.0",
"underscore.string": "~2.3.3", "underscore.string": "~2.3.3",
"pg": "~3.4.1",
"pg-types": "1.4.0",
"express": "~2.5.11", "express": "~2.5.11",
"step": "~0.0.5", "step": "~0.0.5",
"topojson": "1.6.15", "topojson": "0.0.8",
"oauth-client": "0.3.0", "oauth-client": "0.3.0",
"lru-cache":"~2.5.0", "lru-cache":"~2.5.0",
"log4js": "~0.6.16", "log4js": "~0.6.16",
"rollbar": "~0.3.12", "rollbar": "~0.3.2",
"node-statsd": "~0.0.7" "node-statsd": "~0.0.7"
}, },
"devDependencies": { "devDependencies": {
@ -38,7 +37,7 @@
"libxmljs": "~0.8.1" "libxmljs": "~0.8.1"
}, },
"scripts": { "scripts": {
"test": "test/run_tests.sh ${RUNTESTFLAGS} test/unit/*.js test/unit/model/*.js test/unit/sql/*.js test/acceptance/*.js test/acceptance/export/*.js" "test": "test/run_tests.sh ${RUNTESTFLAGS} test/unit/*.js test/unit/model/*.js test/acceptance/*.js test/acceptance/export/*.js"
}, },
"engines": { "node": ">= 0.8.0 < 0.11" } "engines": { "node": ">= 0.8.0 < 0.11" }
} }

View File

@ -1,112 +0,0 @@
require('../helper');
var _ = require('underscore')
, PSQL = require('../../app/models/psql')
, assert = require('assert');
var public_user = global.settings.db_pubuser;
var dbopts_auth = {
host: global.settings.db_host,
port: global.settings.db_port,
user: _.template(global.settings.db_user, {user_id: 1}),
dbname: _.template(global.settings.db_base_name, {user_id: 1}),
pass: _.template(global.settings.db_user_pass, {user_id: 1})
}
var dbopts_anon = _.clone(dbopts_auth);
dbopts_anon.user = global.settings.db_pubuser;
dbopts_anon.pass = global.settings.db_pubuser_pass;
suite('psql', function() {
test('test throws error if no args passed to constructor', function(){
var msg;
try{
var pg = new PSQL();
} catch (err){
msg = err.message;
}
assert.equal(msg, "Incorrect access parameters. If you are accessing via OAuth, please check your tokens are correct. For public users, please ensure your table is published.");
});
test('test private user can execute SELECTS on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "SELECT 1 as test_sum";
pg.query(sql, function(err, result){
assert.ok(!err, err);
assert.equal(result.rows[0].test_sum, 1);
done();
});
});
test('test private user can execute CREATE on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors; CREATE TABLE distributors (id integer, name varchar(40), UNIQUE(name))";
pg.query(sql, function(err, result){
assert.ok(_.isNull(err));
done();
});
});
test('test private user can execute INSERT on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors1; CREATE TABLE distributors1 (id integer, name varchar(40), UNIQUE(name))";
pg.query(sql, function(err, result){
sql = "INSERT INTO distributors1 (id, name) VALUES (1, 'fish')";
pg.query(sql,function(err, result){
assert.deepEqual(result.rows, []);
done();
});
});
});
test('test public user can execute SELECT on enabled tables', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors2; CREATE TABLE distributors2 (id integer, name varchar(40), UNIQUE(name)); GRANT SELECT ON distributors2 TO " + public_user + ";";
pg.query(sql, function(err, result){
pg = new PSQL(dbopts_anon)
pg.query("SELECT count(*) FROM distributors2", function(err, result){
assert.equal(result.rows[0].count, 0);
done();
});
});
});
test('test public user cannot execute INSERT on db', function(done){
var pg = new PSQL(dbopts_auth);
var sql = "DROP TABLE IF EXISTS distributors3; CREATE TABLE distributors3 (id integer, name varchar(40), UNIQUE(name)); GRANT SELECT ON distributors3 TO " + public_user + ";";
pg.query(sql, function(err, result){
pg = new PSQL(dbopts_anon);
pg.query("INSERT INTO distributors3 (id, name) VALUES (1, 'fishy')", function(err, result){
assert.equal(err.message, 'permission denied for relation distributors3');
done();
});
});
});
test('dbkey depends on dbopts', function(){
var opt1 = _.clone(dbopts_anon);
opt1.dbname = 'dbname1';
var pg1 = new PSQL(opt1);
var opt2 = _.clone(dbopts_anon);
opt2.dbname = 'dbname2';
var pg2 = new PSQL(opt2);
assert.ok(pg1.dbkey() !== pg2.dbkey(),
'both PSQL object using same dbkey ' + pg1.dbkey());
assert.ok(_.isString(pg1.dbkey()), "pg1 dbkey is " + pg1.dbkey());
});
test('eventedQuery provisions a cancel mechanism to abort queries', function (done) {
var psql = new PSQL(dbopts_auth);
psql.eventedQuery("SELECT 1 as foo", function(err, query, queryCanceller) {
assert.ok(_.isFunction(queryCanceller));
done();
});
});
});

View File

@ -1,115 +0,0 @@
var _ = require('underscore'),
PSQLWrapper = require('../../../app/sql/psql_wrapper'),
assert = require('assert');
// NOTE: intentionally mixed-case and space-padded
var simpleSql = "\n \tSEleCT * from table1";
suite('psql_wrapper', function() {
test('Windowed SQL with simple select', function(){
var out = new PSQLWrapper(simpleSql).window(1, 0).query();
assert.equal(out, "SELECT * FROM (" + simpleSql + ") AS cdbq_1 LIMIT 1 OFFSET 0");
});
test('Windowed SQL with CTE select', function(){
// NOTE: intentionally mixed-case and space-padded
var cte = "\n \twiTh x as( update test set x=x+1)";
var select = "\n \tSEleCT * from x";
var sql = cte + select;
var out = new PSQLWrapper(sql).window(1, 0).query();
assert.equal(out, cte + "SELECT * FROM (" + select + ") AS cdbq_1 LIMIT 1 OFFSET 0");
});
test('Windowed SQL with CTE update', function(){
// NOTE: intentionally mixed-case and space-padded
var cte = "\n \twiTh a as( update test set x=x+1)";
var upd = "\n \tupdate tost set y=x from x";
var sql = cte + upd;
var out = new PSQLWrapper(sql).window(1, 0).query();
assert.equal(out, sql);
});
test('Windowed SQL with complex CTE and insane quoting', function(){
// NOTE: intentionally mixed-case and space-padded
var cte = "\n \twiTh \"('a\" as( update \"\"\"test)\" set x='x'+1), \")b(\" as ( select ')))\"' from z )";
var sel = "\n \tselect '\"' from x";
var sql = cte + sel;
var out = new PSQLWrapper(sql).window(1, 0).query();
assert.equal(out, cte + "SELECT * FROM (" + sel + ") AS cdbq_1 LIMIT 1 OFFSET 0");
});
test('Different instances return different queries', function() {
var aWrapper = new PSQLWrapper('select 1');
var bWrapper = new PSQLWrapper('select * from databaseB');
assert.notEqual(aWrapper, bWrapper);
assert.notEqual(aWrapper.query(), bWrapper.query(), 'queries should be different');
});
test('Order by SQL with simple select and empty column name returns original query', function() {
var expectedSql = simpleSql;
var outputSql = new PSQLWrapper(simpleSql).orderBy('').query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and no sort order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo"';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo').query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and invalid sort order use no sort order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo"';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo', "BAD_SORT_ORDER").query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and asc order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo" ASC';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo', "asc").query();
assert.equal(outputSql, expectedSql);
});
test('Order by SQL with simple select and DESC order', function() {
var expectedSql = 'SELECT * FROM (' + simpleSql + ') AS cdbq_1 ORDER BY "foo" DESC';
var outputSql = new PSQLWrapper(simpleSql).orderBy('foo', "DESC").query();
assert.equal(outputSql, expectedSql);
});
test('Query with ending semicolon returns without it', function() {
var expectedSql = 'select a, ( a - min(a) over() ) / ( ( max(a) over () - min(a) over () ) / 4 ) as interval from ( select test as a from quantile_test ) as f',
query = expectedSql + ';';
var outputSql = new PSQLWrapper(query).query();
assert.equal(outputSql, expectedSql);
});
test('Several queries with semicolon get only last semicolon removed', function() {
var expectedSql = 'SELECT 1; SELECT 2; SELECT 3',
query = expectedSql + ';';
var outputSql = new PSQLWrapper(query).query();
assert.equal(outputSql, expectedSql);
});
});