diff --git a/NEWS.md b/NEWS.md
index 1d3e3e93..60d1c33c 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,31 @@
+1.3.1 (DD/MM/YY)
+-----
+* Support 'format' and 'filename' params in POST
+* Fix oAuth bug introduced by 'skipfields' param in 1.3.0 (#69)
+
+1.3.0 (DD/MM/YY)
+-----
+* Support for specifying a filename for exports (#64)
+* Support for specifying a list of fields to skip from output (#63)
+* Add 'cache_policy' parameter (#62)
+
+1.2.1 (DD/MM/YY)
+-----
+* Added timeout default to 600 miliseconds in cluster.js
+
+1.2.0 (DD/MM/YY)
+-----
+* New output formats:
+ * ESRI Shapefile (format=shp)
+ * SVG (format=svg)
+ * KML (format=kml)
+* Advertise header presence in CSV Content-Type
+* Fix CSV output with no rows (#60)
+* Use "attachment" Content-Disposition for all output formats (#61)
+* Only use last format parameter when multiple are requested
+* Return a 400 response on unsupported format request
+* Added X-Prototype-Version, X-CSRF-Token to Access-Control-Allow-Headers
+
1.1.0 (30/10/12)
-----
* Fixed problem in cluster2 with pidfile name
diff --git a/app/controllers/app.js b/app/controllers/app.js
index 9e9f33bc..21599e5f 100755
--- a/app/controllers/app.js
+++ b/app/controllers/app.js
@@ -14,6 +14,9 @@
// eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table
//
//
+
+var path = require('path');
+
var express = require('express')
, app = express.createServer(
express.logger({
@@ -23,6 +26,10 @@ var express = require('express')
, Step = require('step')
, csv = require('csv')
, crypto = require('crypto')
+ , fs = require('fs')
+ , zlib = require('zlib')
+ , util = require('util')
+ , spawn = require('child_process').spawn
, Meta = require(global.settings.app_root + '/app/models/metadata')
, oAuth = require(global.settings.app_root + '/app/models/oauth')
, PSQL = require(global.settings.app_root + '/app/models/psql')
@@ -38,26 +45,64 @@ app.all('/api/v1/sql', function(req, res) { handleQuery(req, res) } );
app.all('/api/v1/sql.:f', function(req, res) { handleQuery(req, res) } );
app.get('/api/v1/cachestatus', function(req, res) { handleCacheStatus(req, res) } );
+// Return true of the given query may write to the database
+//
+// NOTE: this is a fuzzy check, the return could be true even
+// if the query doesn't really write anything.
+// But you can be pretty sure of a false return.
+//
+function queryMayWrite(sql) {
+ var mayWrite = false;
+ var pattern = RegExp("(alter|insert|update|delete|create|drop)", "i");
+ if ( pattern.test(sql) ) {
+ mayWrite = true;
+ }
+ return mayWrite;
+}
+
+// Return database username from user_id
+// NOTE: a "null" user_id is a request to use the public user
+function userid_to_dbuser(user_id) {
+ if ( _.isString(user_id) )
+ return _.template(global.settings.db_user, {user_id: user_id});
+ return "publicuser" // FIXME: make configurable
+};
+
+function sanitize_filename(filename) {
+ filename = path.basename(filename, path.extname(filename));
+ filename = filename.replace(/[;()\[\]<>'"\s]/g, '_');
+ //console.log("Sanitized: " + filename);
+ return filename;
+}
+
// request handlers
function handleQuery(req, res) {
+ var supportedFormats = ['json', 'geojson', 'csv', 'svg', 'shp', 'kml'];
+ var svg_width = 1024.0;
+ var svg_height = 768.0;
+
// extract input
var body = (req.body) ? req.body : {};
var sql = req.query.q || body.q; // HTTP GET and POST store in different vars
var api_key = req.query.api_key || body.api_key;
- var database = req.query.database; // TODO: Depricate
+ var database = req.query.database; // TODO: Deprecate
var limit = parseInt(req.query.rows_per_page);
var offset = parseInt(req.query.page);
- var format = req.query.format;
- var dp = req.query.dp; // decimal point digits (defaults to 6)
- var gn = "the_geom"; // TODO: read from configuration file
- var svg_width = 1024.0;
- var svg_height = 768.0;
-
+ var requestedFormat = req.query.format || body.format;
+ var format = _.isArray(requestedFormat) ? _.last(requestedFormat) : requestedFormat;
+ var requestedFilename = req.query.filename || body.filename
+ var filename = requestedFilename;
+ var requestedSkipfields = req.query.skipfields || body.skipfields;
+ var skipfields = requestedSkipfields ? requestedSkipfields.split(',') : [];
+ var dp = req.query.dp || body.dp; // decimal point digits (defaults to 6)
+ var gn = "the_geom"; // TODO: read from configuration file
+ var user_id;
// sanitize and apply defaults to input
dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp;
- format = (format === "" || _.isUndefined(format)) ? null : format.toLowerCase();
+ format = (format === "" || _.isUndefined(format)) ? 'json' : format.toLowerCase();
+ filename = (filename === "" || _.isUndefined(filename)) ? 'cartodb-query' : sanitize_filename(filename);
sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
database = (database === "" || _.isUndefined(database)) ? null : database;
limit = (_.isNumber(limit)) ? limit : null;
@@ -67,6 +112,10 @@ function handleQuery(req, res) {
var start = new Date().getTime();
try {
+
+ if ( -1 === supportedFormats.indexOf(format) )
+ throw new Error("Invalid format: " + format);
+
if (!_.isString(sql)) throw new Error("You must indicate a sql query");
// initialise MD5 key of sql for cache lookups
@@ -75,6 +124,8 @@ function handleQuery(req, res) {
// placeholder for connection
var pg;
+ var authenticated;
+
// 1. Get database from redis via the username stored in the host header subdomain
// 2. Run the request through OAuth to get R/W user id if signed
// 3. Get the list of tables affected by the query
@@ -108,17 +159,20 @@ function handleQuery(req, res) {
oAuth.verifyRequest(req, this);
}
},
- function queryExplain(err, user_id){
+ function queryExplain(err, data){
if (err) throw err;
+ user_id = data;
// store postgres connection
pg = new PSQL(user_id, database, limit, offset);
+ authenticated = ! _.isNull(user_id);
+
// get all the tables from Cache or SQL
if (!_.isNull(tableCache[sql_md5]) && !_.isUndefined(tableCache[sql_md5])){
tableCache[sql_md5].hits++;
return true;
- } else{
- pg.query("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)", this);
+ } else {
+ pg.query("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)", this);
}
},
function queryResult(err, result){
@@ -127,15 +181,18 @@ function handleQuery(req, res) {
// store explain result in local Cache
if (_.isUndefined(tableCache[sql_md5])){
tableCache[sql_md5] = result;
+ tableCache[sql_md5].may_write = queryMayWrite(sql);
tableCache[sql_md5].hits = 1; //initialise hit counter
}
// TODO: refactor formats to external object
if (format === 'geojson'){
sql = ['SELECT *, ST_AsGeoJSON(the_geom,',dp,') as the_geom FROM (', sql, ') as foo'].join("");
- } else if (format === 'svg'){
+ } else if (format === 'shp') {
+ return null;
+ } else if (format === 'svg') {
var svg_ratio = svg_width/svg_height;
- sql = 'WITH source AS ( ' + sql + '), extent AS ( '
+ sql = 'WITH source AS ( ' + sql + '), extent AS ( '
+ ' SELECT ST_Extent(' + gn + ') AS e FROM source '
+ '), extent_info AS ( SELECT e, '
+ 'st_xmin(e) as ex0, st_ymax(e) as ey0, '
@@ -161,23 +218,40 @@ function handleQuery(req, res) {
function setHeaders(err, result){
if (err) throw err;
- // configure headers for geojson/CSV
- res.header("Content-Disposition", getContentDisposition(format));
+ // configure headers for given format
+ var use_inline = !requestedFormat && !requestedFilename;
+ res.header("Content-Disposition", getContentDisposition(format, filename, use_inline));
res.header("Content-Type", getContentType(format));
// allow cross site post
setCrossDomain(res);
// set cache headers
- res.header('Last-Modified', new Date().toUTCString());
- res.header('Cache-Control', 'no-cache,max-age=3600,must-revalidate, public');
- res.header('X-Cache-Channel', generateCacheKey(database, tableCache[sql_md5]));
+ res.header('X-Cache-Channel', generateCacheKey(database, tableCache[sql_md5], authenticated));
+ var cache_policy = req.query.cache_policy;
+ if ( cache_policy == 'persist' ) {
+ res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+ } else {
+ // TODO: set ttl=0 when tableCache[sql_md5].may_write is true ?
+ var ttl = 3600;
+ res.header('Last-Modified', new Date().toUTCString());
+ res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate,public');
+ }
+
return result;
},
function packageResults(err, result){
if (err) throw err;
+ if ( skipfields.length ){
+ for ( var i=0; i');
}
- if ( ! bbox ) {
+ if ( ! bbox ) {
// Parse layer extent: "BOX(x y, X Y)"
// NOTE: the name of the extent field is
// determined by the same code adding the
@@ -291,9 +368,9 @@ function toSVG(rows, gn, callback){
bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/);
bbox = {
xmin: parseFloat(bbox[1]),
- ymin: parseFloat(bbox[2]),
+ ymin: parseFloat(bbox[2]),
xmax: parseFloat(bbox[3]),
- ymax: parseFloat(bbox[4])
+ ymax: parseFloat(bbox[4])
};
}
});
@@ -320,7 +397,7 @@ function toSVG(rows, gn, callback){
bbox.width = bbox.xmax - bbox.xmin;
bbox.height = bbox.ymax - bbox.ymin;
root_tag += 'viewBox="' + bbox.xmin + ' ' + (-bbox.ymax) + ' '
- + bbox.width + ' ' + bbox.height + '" ';
+ + bbox.width + ' ' + bbox.height + '" ';
}
root_tag += 'style="fill-opacity:' + fill_opacity
+ '; stroke:' + stroke_color
@@ -343,7 +420,7 @@ function toSVG(rows, gn, callback){
function toCSV(data, res, callback){
try{
// pull out keys for column headers
- var columns = _.keys(data.rows[0]);
+ var columns = data.rows.length ? _.keys(data.rows[0]) : [];
// stream the csv out over http
csv()
@@ -355,7 +432,289 @@ function toCSV(data, res, callback){
}
}
-function getContentDisposition(format){
+// Internal function usable by all OGR-driven outputs
+function toOGR(dbname, user_id, gcol, sql, skipfields, res, out_format, out_filename, callback) {
+ var ogr2ogr = 'ogr2ogr'; // FIXME: make configurable
+ var dbhost = global.settings.db_host;
+ var dbport = global.settings.db_port;
+ var dbuser = userid_to_dbuser(user_id);
+ var dbpass = ''; // turn into a parameter..
+
+ var tmpdir = '/tmp'; // FIXME: make configurable
+ var columns = [];
+
+ Step (
+
+ function fetchColumns() {
+ var colsql = 'SELECT * FROM (' + sql + ') as _cartodbsqlapi LIMIT 1';
+ var pg = new PSQL(user_id, dbname, 1, 0);
+ pg.query(colsql, this);
+ },
+ function spawnDumper(err, result) {
+ if (err) throw err;
+
+ if ( ! result.rows.length )
+ throw new Error("Query returns no rows");
+
+ // Skip system columns
+ for (var k in result.rows[0]) {
+ if ( skipfields.indexOf(k) != -1 ) continue;
+ if ( k == "the_geom_webmercator" ) continue;
+ columns.push('"' + k + '"');
+ }
+ //console.log(columns.join(','));
+
+ var next = this;
+
+ sql = 'SELECT ' + columns.join(',')
+ + ' FROM (' + sql + ') as _cartodbsqlapi';
+
+ var child = spawn(ogr2ogr, [
+ '-f', out_format,
+ out_filename,
+ "PG:host=" + dbhost
+ + " user=" + dbuser
+ + " dbname=" + dbname
+ + " password=" + dbpass
+ + " tables=fake" // trick to skip query to geometry_columns
+ + "",
+ '-sql', sql
+ ]);
+
+/*
+console.log(['ogr2ogr',
+ '-f', out_format,
+ out_filename,
+ "'PG:host=" + dbhost
+ + " user=" + dbuser
+ + " dbname=" + dbname
+ + " password=" + dbpass
+ + " tables=fake" // trick to skip query to geometry_columns
+ + "'",
+ '-sql "', sql, '"'].join(' '));
+*/
+
+ var stdout = '';
+ child.stdout.on('data', function(data) {
+ stdout += data;
+ //console.log('stdout: ' + data);
+ });
+
+ var stderr = '';
+ child.stderr.on('data', function(data) {
+ stderr += data;
+ console.log('ogr2ogr stderr: ' + data);
+ });
+
+ child.on('exit', function(code) {
+ if ( code ) {
+ next(new Error("ogr2ogr returned an error (error code " + code + ")\n" + stderr));
+ } else {
+ next(null);
+ }
+ });
+ },
+ function finish(err) {
+ callback(err);
+ }
+ );
+}
+
+function toSHP(dbname, user_id, gcol, sql, skipfields, filename, res, callback) {
+ var zip = 'zip'; // FIXME: make configurable
+ var tmpdir = '/tmp'; // FIXME: make configurable
+ var outdirpath = tmpdir + '/sqlapi-shapefile-' + generateMD5(sql);
+ var shapefile = outdirpath + '/' + filename + '.shp';
+
+ // TODO: following tests:
+ // - fetch with no auth [done]
+ // - fetch with auth [done]
+ // - fetch same query concurrently
+ // - fetch query with no "the_geom" column
+
+ // TODO: Check if the file already exists
+ // (should mean another export of the same query is in progress)
+
+ Step (
+
+ function createOutDir() {
+ fs.mkdir(outdirpath, 0777, this);
+ },
+ function spawnDumper(err) {
+ if ( err ) {
+ if ( err.code == 'EEXIST' ) {
+ // TODO: this could mean another request for the same
+ // resource is in progress, in which case we might want
+ // to queue the response to after it's completed...
+ console.log("Reusing existing SHP output directory for query: " + sql);
+ } else {
+ throw err;
+ }
+ }
+ toOGR(dbname, user_id, gcol, sql, skipfields, res, 'ESRI Shapefile', shapefile, this);
+ },
+ function zipAndSendDump(err) {
+ if ( err ) throw err;
+
+ var next = this;
+ var dir = outdirpath;
+
+ var zipfile = dir + '.zip';
+
+ var child = spawn(zip, ['-qrj', '-', dir ]);
+
+ child.stdout.on('data', function(data) {
+ res.write(data);
+ });
+
+ var stderr = '';
+ child.stderr.on('data', function(data) {
+ stderr += data;
+ console.log('zip stderr: ' + data);
+ });
+
+ child.on('exit', function(code) {
+ if (code) {
+ res.statusCode = 500;
+ //res.send(stderr);
+ }
+ //console.log("Zip complete, zip return code was " + code);
+ next(null);
+ });
+
+ },
+ function cleanupDir(topError) {
+
+ var next = this;
+
+ //console.log("Cleaning up " + outdirpath);
+
+ // Unlink the dir content
+ var unlinkall = function(dir, files, finish) {
+ var f = files.shift();
+ if ( ! f ) { finish(null); return; }
+ var fn = dir + '/' + f;
+ fs.unlink(fn, function(err) {
+ if ( err ) {
+ console.log("Unlinking " + fn + ": " + err);
+ finish(err);
+ }
+ else unlinkall(dir, files, finish)
+ });
+ }
+ fs.readdir(outdirpath, function(err, files) {
+ if ( err ) {
+ if ( err.code != 'ENOENT' ) {
+ next(new Error([topError, err].join('\n')));
+ } else {
+ next(topError);
+ }
+ } else {
+ unlinkall(outdirpath, files, function(err) {
+ fs.rmdir(outdirpath, function(err) {
+ if ( err ) console.log("Removing dir " + path + ": " + err);
+ next(topError);
+ });
+ });
+ }
+ });
+ },
+ function finish(err) {
+ if ( err ) callback(err);
+ else {
+ res.end();
+ callback(null);
+ }
+
+ }
+ );
+}
+
+function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) {
+ var zip = 'zip'; // FIXME: make configurable
+ var tmpdir = '/tmp'; // FIXME: make configurable
+ var outdirpath = tmpdir + '/sqlapi-kmloutput-' + generateMD5(sql);
+ var dumpfile = outdirpath + '/cartodb-query.kml';
+
+ // TODO: following tests:
+ // - fetch with no auth
+ // - fetch with auth
+ // - fetch same query concurrently
+ // - fetch query with no "the_geom" column
+
+ Step (
+
+ function createOutDir() {
+ fs.mkdir(outdirpath, 0777, this);
+ },
+ function spawnDumper(err) {
+ if ( err ) {
+ if ( err.code == 'EEXIST' ) {
+ // TODO: this could mean another request for the same
+ // resource is in progress, in which case we might want
+ // to queue the response to after it's completed...
+ console.log("Reusing existing KML output directory for query: " + sql);
+ } else {
+ throw err;
+ }
+ }
+ toOGR(dbname, user_id, gcol, sql, skipfields, res, 'KML', dumpfile, this);
+ },
+ function sendResults(err) {
+
+ if ( ! err ) {
+ var stream = fs.createReadStream(dumpfile);
+ util.pump(stream, res);
+ }
+
+ // cleanup output dir (should be safe to unlink)
+ var topError = err;
+ var next = this;
+
+ //console.log("Cleaning up " + outdirpath);
+
+ // Unlink the dir content
+ var unlinkall = function(dir, files, finish) {
+ var f = files.shift();
+ if ( ! f ) { finish(null); return; }
+ var fn = dir + '/' + f;
+ fs.unlink(fn, function(err) {
+ if ( err ) {
+ console.log("Unlinking " + fn + ": " + err);
+ finish(err);
+ }
+ else unlinkall(dir, files, finish)
+ });
+ }
+ fs.readdir(outdirpath, function(err, files) {
+ if ( err ) {
+ if ( err.code != 'ENOENT' ) {
+ next(new Error([topError, err].join('\n')));
+ } else {
+ next(topError);
+ }
+ } else {
+ unlinkall(outdirpath, files, function(err) {
+ fs.rmdir(outdirpath, function(err) {
+ if ( err ) console.log("Removing dir " + path + ": " + err);
+ next(topError);
+ });
+ });
+ }
+ });
+ },
+ function finish(err) {
+ if ( err ) callback(err);
+ else {
+ res.end();
+ callback(null);
+ }
+
+ }
+ );
+}
+
+function getContentDisposition(format, filename, inline) {
var ext = 'json';
if (format === 'geojson'){
ext = 'geojson';
@@ -366,28 +725,44 @@ function getContentDisposition(format){
else if (format === 'svg'){
ext = 'svg';
}
+ else if (format === 'shp'){
+ ext = 'zip';
+ }
+ else if (format === 'kml'){
+ ext = 'kml';
+ }
var time = new Date().toUTCString();
- return 'inline; filename=cartodb-query.' + ext + '; modification-date="' + time + '";';
+ return ( inline ? 'inline' : 'attachment' ) +'; filename=' + filename + '.' + ext + '; modification-date="' + time + '";';
}
function getContentType(format){
var type = "application/json; charset=utf-8";
if (format === 'csv'){
- type = "text/csv; charset=utf-8";
+ type = "text/csv; charset=utf-8; header=present";
}
else if (format === 'svg'){
type = "image/svg+xml; charset=utf-8";
}
+ else if (format === 'shp'){
+ type = "application/zip; charset=utf-8";
+ }
+ else if (format === 'kml'){
+ type = "application/kml; charset=utf-8";
+ }
return type;
}
function setCrossDomain(res){
res.header("Access-Control-Allow-Origin", "*");
- res.header("Access-Control-Allow-Headers", "X-Requested-With");
+ res.header("Access-Control-Allow-Headers", "X-Requested-With, X-Prototype-Version, X-CSRF-Token");
}
-function generateCacheKey(database,tables){
- return database + ":" + tables.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
+function generateCacheKey(database,tables,is_authenticated){
+ if ( is_authenticated && tables.may_write ) {
+ return "NONE";
+ } else {
+ return database + ":" + tables.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
+ }
}
function generateMD5(data){
diff --git a/app/models/apikey_auth.js b/app/models/apikey_auth.js
index 9cc54846..3d41dd97 100644
--- a/app/models/apikey_auth.js
+++ b/app/models/apikey_auth.js
@@ -88,7 +88,7 @@ module.exports = (function() {
* Get privacy for cartodb table
*
* @param req - standard req object. Importantly contains table and host information
- * @param callback - user_id if ok, null if auth fails
+ * @param callback - err, user_id (null if no auth)
*/
me.verifyRequest = function(req, callback) {
var that = this;
@@ -108,8 +108,8 @@ module.exports = (function() {
}
},
function (err, user_id){
- if (err) throw err;
- callback(false, user_id);
+ if (err) callback(err);
+ else callback(false, user_id);
}
);
};
diff --git a/app/models/metadata.js b/app/models/metadata.js
index 36abf647..518d3a5f 100644
--- a/app/models/metadata.js
+++ b/app/models/metadata.js
@@ -72,4 +72,4 @@ module.exports = function() {
};
return me;
-}();
\ No newline at end of file
+}();
diff --git a/cluster.js b/cluster.js
index 30417979..97f1dd98 100755
--- a/cluster.js
+++ b/cluster.js
@@ -32,7 +32,8 @@ var cluster = new Cluster({
port: global.settings.node_port,
host: global.settings.node_host,
monHost: global.settings.node_host,
- monPort: global.settings.node_port+1
+ monPort: global.settings.node_port+1,
+ timeout: 600000
});
cluster.listen(function(cb) {
diff --git a/doc/API.md b/doc/API.md
index ceb72c09..d61198a4 100644
--- a/doc/API.md
+++ b/doc/API.md
@@ -12,7 +12,14 @@ Supported query string parameters:
'format': Specifies which format to use for the response.
Supported formats: JSON (the default), GeoJSON,
- CSV, SVG
+ CSV, SVG, SHP
+
+ 'filename': Sets the filename to use for the query result
+ file attachment
+
+ 'skipfields':
+ Comma separate list of fields that are not wanted
+ in output. Only useful with "SELECT *" queries.
'dp': Number of digits after the decimal point.
Only affects format=GeoJSON.
@@ -20,6 +27,13 @@ Supported query string parameters:
'api_key': Needed to authenticate in order to modify the database.
+ 'cache_policy':
+ Set to "persist" to have the server send an Cache-Control
+ header requesting caching devices to keep the response
+ cached as much as possible. This is best used with a
+ timestamp value in cache_buster for manual control of
+ updates.
+
Response formats
----------------
diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index 2862185d..6bf900b9 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -1,9 +1,9 @@
{
"name": "cartodb_api",
- "version": "1.1.0",
+ "version": "1.3.0",
"dependencies": {
"cluster2": {
- "version": "0.3.5-cdb01",
+ "version": "0.3.5-cdb02",
"from": "git://github.com/CartoDB/cluster2.git#cdb_production",
"dependencies": {
"ejs": {
@@ -218,6 +218,17 @@
"csv": {
"version": "0.0.13"
},
+ "zipfile": {
+ "version": "0.3.2"
+ },
+ "libxmljs": {
+ "version": "0.6.1",
+ "dependencies": {
+ "bindings": {
+ "version": "1.0.0"
+ }
+ }
+ },
"mocha": {
"version": "1.2.1",
"dependencies": {
diff --git a/package.json b/package.json
index fc18f7ed..4dcfc82b 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
"private": true,
"name": "cartodb_api",
"description": "high speed SQL api for cartodb",
- "version": "1.1.0",
+ "version": "1.3.1",
"author": {
"name": "Simon Tokumine, Sandro Santilli, Vizzuality",
"url": "http://vizzuality.com",
@@ -23,7 +23,9 @@
"csv":"0.0.13"
},
"devDependencies": {
- "mocha": "1.2.1"
+ "mocha": "1.2.1",
+ "zipfile": "~0.3.2",
+ "libxmljs": "~0.6.1"
},
"scripts": {
"test": "test/run_tests.sh"
diff --git a/test/acceptance/app.test.js b/test/acceptance/app.test.js
index 6dace8a6..08f8b578 100644
--- a/test/acceptance/app.test.js
+++ b/test/acceptance/app.test.js
@@ -14,17 +14,23 @@
require('../helper');
require('../support/assert');
+
var app = require(global.settings.app_root + '/app/controllers/app')
, assert = require('assert')
, querystring = require('querystring')
- , _ = require('underscore');
+ , _ = require('underscore')
+ , zipfile = require('zipfile')
+ , fs = require('fs')
+ , libxmljs = require('libxmljs')
+ ;
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
suite('app.test', function() {
-var real_oauth_header = 'OAuth realm="http://vizzuality.testhost.lan/",oauth_consumer_key="fZeNGv5iYayvItgDYHUbot1Ukb5rVyX6QAg8GaY2",oauth_token="l0lPbtP68ao8NfStCiA3V3neqfM03JKhToxhUQTR",oauth_signature_method="HMAC-SHA1", oauth_signature="o4hx4hWP6KtLyFwggnYB4yPK8xI%3D",oauth_timestamp="1313581372",oauth_nonce="W0zUmvyC4eVL8cBd4YwlH1nnPTbxW0QBYcWkXTwe4",oauth_version="1.0"';
+var expected_cache_control = 'no-cache,max-age=3600,must-revalidate,public';
+var expected_cache_control_persist = 'public,max-age=31536000';
// use dec_sep for internationalization
var checkDecimals = function(x, dec_sep){
@@ -54,10 +60,27 @@ test('GET /api/v1/sql with SQL parameter on SELECT only. No oAuth included ', fu
method: 'GET'
},{ }, function(res) {
assert.equal(res.statusCode, 200, res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:untitle_table_4');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
done();
});
});
+test('cache_policy=persist', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db&cache_policy=persist',
+ method: 'GET'
+ },{ }, function(res) {
+ assert.equal(res.statusCode, 200, res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:untitle_table_4');
+ assert.equal(res.headers['cache-control'], expected_cache_control_persist);
+ done();
+ });
+});
test('GET /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers', function(done){
assert.response(app, {
@@ -70,6 +93,22 @@ test('GET /api/v1/sql with SQL parameter on SELECT only. no database param, just
});
});
+test('GET /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers. Authenticated.',
+function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20cartodb_id*2%20FROM%20untitle_table_4&api_key=1234',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res) {
+ assert.equal(res.statusCode, 200, res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:untitle_table_4');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ done();
+ });
+});
+
test('POST /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers', function(done){
assert.response(app, {
@@ -129,6 +168,10 @@ test('INSERT returns affected rows', function(done){
assert.ok(out.hasOwnProperty('time'));
assert.equal(out.total_rows, 2);
assert.equal(out.rows.length, 0);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
done();
});
});
@@ -151,6 +194,10 @@ test('UPDATE returns affected rows', function(done){
assert.ok(out.hasOwnProperty('time'));
assert.equal(out.total_rows, 2);
assert.equal(out.rows.length, 0);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
done();
});
});
@@ -173,6 +220,10 @@ test('DELETE returns affected rows', function(done){
assert.ok(out.hasOwnProperty('time'));
assert.equal(out.total_rows, 2);
assert.equal(out.rows.length, 0);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
done();
});
});
@@ -261,11 +312,306 @@ test('GET /api/v1/sql with SQL parameter on DROP DATABASE only.header based db -
});
});
+test('CREATE TABLE with GET and auth', function(done){
+ assert.response(app, {
+ url: "/api/v1/sql?" + querystring.stringify({
+ q: 'CREATE TABLE test_table(a int)',
+ api_key: 1234
+ }),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{}, function(res) {
+ assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ done();
+ });
+});
+
+// TODO: test COPY
+//test('COPY TABLE with GET and auth', function(done){
+// assert.response(app, {
+// url: "/api/v1/sql?" + querystring.stringify({
+// q: 'COPY TABLE test_table FROM stdin; 1\n\\.\n',
+// api_key: 1234
+// }),
+// headers: {host: 'vizzuality.cartodb.com'},
+// method: 'GET'
+// },{}, function(res) {
+// assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+// // Check cache headers
+// // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+// assert.equal(res.headers['x-cache-channel'], 'NONE');
+// assert.equal(res.headers['cache-control'], expected_cache_control);
+// done();
+// });
+//});
+
+test('ALTER TABLE with GET and auth', function(done){
+ assert.response(app, {
+ url: "/api/v1/sql?" + querystring.stringify({
+ q: 'ALTER TABLE test_table ADD b int',
+ api_key: 1234
+ }),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{}, function(res) {
+ assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ done();
+ });
+});
+
+test('DROP TABLE with GET and auth', function(done){
+ assert.response(app, {
+ url: "/api/v1/sql?" + querystring.stringify({
+ q: 'DROP TABLE test_table',
+ api_key: 1234
+ }),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{}, function(res) {
+ assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ done();
+ });
+});
+
+test('CREATE FUNCTION with GET and auth', function(done){
+ assert.response(app, {
+ url: "/api/v1/sql?" + querystring.stringify({
+ q: 'CREATE FUNCTION create_func_test(a int) RETURNS INT AS \'SELECT 1\' LANGUAGE \'sql\'',
+ api_key: 1234
+ }),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{}, function(res) {
+ assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ done();
+ });
+});
+
+test('DROP FUNCTION with GET and auth', function(done){
+ assert.response(app, {
+ url: "/api/v1/sql?" + querystring.stringify({
+ q: 'DROP FUNCTION create_func_test(a int)',
+ api_key: 1234
+ }),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{}, function(res) {
+ assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'NONE');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ done();
+ });
+});
+
+test('sends a 400 when an unsupported format is requested', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=unknown',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 400, res.body);
+ assert.deepEqual(JSON.parse(res.body), {"error":[ "Invalid format: unknown" ]});
+ done();
+ });
+});
+
+test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var ct = res.header('Content-Type');
+ assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^inline/.test(cd), 'Default format is not disposed inline: ' + cd);
+ assert.equal(true, /filename=cartodb-query.json/gi.test(cd), 'Unexpected JSON filename: ' + cd);
+ done();
+ });
+});
+
+test('POST /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql',
+ data: querystring.stringify({q: "SELECT * FROM untitle_table_4" }),
+ headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+ method: 'POST'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var ct = res.header('Content-Type');
+ assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^inline/.test(cd), 'Default format is not disposed inline: ' + cd);
+ assert.equal(true, /filename=cartodb-query.json/gi.test(cd), 'Unexpected JSON filename: ' + cd);
+ done();
+ });
+});
+
+test('GET /api/v1/sql with SQL parameter and no format, but a filename', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&filename=x',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var ct = res.header('Content-Type');
+ assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'Format with filename is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=x.json/gi.test(cd), 'Unexpected JSON filename: ' + cd);
+ done();
+ });
+});
+
+test('field named "the_geom_webmercator" is not skipped by default', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var row0 = JSON.parse(res.body).rows[0];
+ var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':1, 'the_geom_webmercator':1};
+ for ( var f in checkfields ) {
+ if ( checkfields[f] ) {
+ assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+ } else {
+ assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+ }
+ }
+ done();
+ });
+});
+
+test('skipfields controls included fields', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&skipfields=the_geom_webmercator,cartodb_id,unexistant',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var row0 = JSON.parse(res.body).rows[0];
+ var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':0};
+ for ( var f in checkfields ) {
+ if ( checkfields[f] ) {
+ assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+ } else {
+ assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+ }
+ }
+ done();
+ });
+});
+
+test('GET /api/v1/sql ensure cross domain set on errors', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*gadfgadfg%20FROM%20untitle_table_4',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{
+ status: 400
+ }, function(res){
+ var cd = res.header('Access-Control-Allow-Origin');
+ assert.equal(cd, '*');
+ done();
+ });
+});
+
+test('cannot GET system tables', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20pg_attribute',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{
+ status: 403
+ }, function() { done(); });
+});
+
+test('GET decent error if domain is incorrect', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
+ headers: {host: 'vizzualinot.cartodb.com'},
+ method: 'GET'
+ },{
+ status: 404
+ }, function(res){
+ var result = JSON.parse(res.body);
+ assert.equal(result.error[0],"Sorry, we can't find this CartoDB. Please check that you have entered the correct domain.");
+ done();
+ });
+});
+
+test('GET decent error if SQL is broken', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?' + querystring.stringify({q:
+ 'SELECT star FROM this and that'
+ }),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{}, function(res){
+ assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+ var result = JSON.parse(res.body);
+ // NOTE: actual error message may be slighly different, possibly worth a regexp here
+ assert.equal(result.error[0], 'syntax error at or near "and"');
+ done();
+ });
+});
+
+// GEOJSON tests
+
test('GET /api/v1/sql with SQL parameter and geojson format, ensuring content-disposition set to geojson', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
+ done();
+ });
+});
+
+test('POST /api/v1/sql with SQL parameter and geojson format, ensuring content-disposition set to geojson', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql',
+ data: querystring.stringify({q: "SELECT * FROM untitle_table_4", format: 'geojson' }),
+ headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+ method: 'POST'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
+ done();
+ });
+});
+
+test('uses the last format parameter when multiple are used', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?format=csv&q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
@@ -274,6 +620,218 @@ test('GET /api/v1/sql with SQL parameter and geojson format, ensuring content-di
});
});
+test('uses custom filename', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&filename=x',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /filename=x.geojson/gi.test(cd), cd);
+ done();
+ });
+});
+
+test('does not include the_geom and the_geom_webmercator properties by default', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var parsed_body = JSON.parse(res.body);
+ var row0 = parsed_body.features[0].properties;
+ var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':0, 'the_geom_webmercator':0};
+ for ( var f in checkfields ) {
+ if ( checkfields[f] ) {
+ assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+ } else {
+ assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+ }
+ }
+ done();
+ });
+});
+
+test('skipfields controls fields included in GeoJSON output', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&skipfields=unexistant,cartodb_id',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var parsed_body = JSON.parse(res.body);
+ var row0 = parsed_body.features[0].properties;
+ var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':0, 'the_geom_webmercator':0};
+ for ( var f in checkfields ) {
+ if ( checkfields[f] ) {
+ assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+ } else {
+ assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+ }
+ }
+ done();
+ });
+});
+
+
+test('GET /api/v1/sql as geojson limiting decimal places', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?' + querystring.stringify({
+ q: 'SELECT ST_MakePoint(0.123,2.3456) as the_geom',
+ format: 'geojson',
+ dp: '1'}),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var result = JSON.parse(res.body);
+ assert.equal(1, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
+ done();
+ });
+});
+
+test('GET /api/v1/sql as geojson with default dp as 6', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?' + querystring.stringify({
+ q: 'SELECT ST_MakePoint(0.12345678,2.3456787654) as the_geom',
+ format: 'geojson'}),
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var result = JSON.parse(res.body);
+ assert.equal(6, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
+ done();
+ });
+});
+
+
+// CSV tests
+
+test('CSV format', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
+ var ct = res.header('Content-Type');
+ assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
+ done();
+ });
+});
+
+test('CSV format from POST', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql',
+ data: querystring.stringify({q: "SELECT * FROM untitle_table_4 LIMIT 1", format: 'csv'}),
+ headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+ method: 'POST'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
+ var ct = res.header('Content-Type');
+ assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
+ done();
+ });
+});
+
+test('CSV format, custom filename', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv&filename=mycsv.csv',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=mycsv.csv/gi.test(cd), cd);
+ var ct = res.header('Content-Type');
+ assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
+ var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
+ var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':1, 'the_geom_webmercator':1};
+ for ( var f in checkfields ) {
+ var idx = row0.indexOf(f);
+ if ( checkfields[f] ) {
+ assert.ok(idx != -1, "result does not include '" + f + "'");
+ } else {
+ assert.ok(idx == -1, "result includes '" + f + "' ("+idx+")");
+ }
+ }
+ done();
+ });
+});
+
+test('skipfields controls fields included in CSV output', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv&skipfields=unexistant,cartodb_id',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
+ var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':1};
+ for ( var f in checkfields ) {
+ var idx = row0.indexOf(f);
+ if ( checkfields[f] ) {
+ assert.ok(idx != -1, "result does not include '" + f + "'");
+ } else {
+ assert.ok(idx == -1, "result includes '" + f + "' ("+idx+")");
+ }
+ }
+ done();
+ });
+});
+
+test('GET /api/v1/sql as csv', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20cartodb_id,ST_AsEWKT(the_geom)%20as%20geom%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var body = "cartodb_id,geom\r\n1,SRID=4326;POINT(-3.699732 40.423012)";
+ assert.equal(body, res.body);
+ done();
+ });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/60
+test('GET /api/v1/sql as csv with no rows', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20true%20WHERE%20false&format=csv',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var body = "";
+ assert.equal(body, res.body);
+ done();
+ });
+});
+
+test('GET /api/v1/sql as csv, properly escaped', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20cartodb_id,%20address%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var body = 'cartodb_id,address\r\n1,"Calle de Pérez Galdós 9, Madrid, Spain"';
+ assert.equal(body, res.body);
+ done();
+ });
+});
+
+// SVG tests
+
test('GET /api/v1/sql with SVG format', function(done){
var query = querystring.stringify({
q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
@@ -294,6 +852,49 @@ test('GET /api/v1/sql with SVG format', function(done){
});
});
+test('POST /api/v1/sql with SVG format', function(done){
+ var query = querystring.stringify({
+ q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
+ format: "svg"
+ });
+ assert.response(app, {
+ url: '/api/v1/sql',
+ data: query,
+ headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+ method: 'POST'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd);
+ assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+ assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
+ assert.ok( res.body.indexOf('') > 0, res.body );
+ // TODO: test viewBox
+ done();
+ });
+});
+
+test('GET /api/v1/sql with SVG format and custom filename', function(done){
+ var query = querystring.stringify({
+ q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
+ format: "svg",
+ filename: 'mysvg'
+ });
+ assert.response(app, {
+ url: '/api/v1/sql?' + query,
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.ok(/filename=mysvg.svg/gi.test(cd), cd);
+ assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
+ assert.ok( res.body.indexOf('') > 0, res.body );
+ // TODO: test viewBox
+ done();
+ });
+});
+
test('GET /api/v1/sql with SVG format and centered point', function(done){
var query = querystring.stringify({
q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ",
@@ -341,6 +942,7 @@ test('GET /api/v1/sql with SVG format and trimmed decimals', function(done){
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd);
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('') > 0, res.body );
@@ -350,137 +952,252 @@ test('GET /api/v1/sql with SVG format and trimmed decimals', function(done){
});
});
-test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){
+
+// SHP tests
+
+test('SHP format, unauthenticated', function(done){
assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
+ headers: {host: 'vizzuality.cartodb.com'},
+ encoding: 'binary',
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
+ var tmpfile = '/tmp/myshape.zip';
+ var err = fs.writeFileSync(tmpfile, res.body, 'binary');
+ if (err) { done(err); return }
+ var zf = new zipfile.ZipFile(tmpfile);
+ assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+ assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+ assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+ // missing SRID, so no PRJ (TODO: add ?)
+ //assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+ // TODO: check DBF contents
+ fs.unlinkSync(tmpfile);
+ done();
+ });
+});
+
+test('SHP format, unauthenticated, POST', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql',
+ data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
+ headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+ method: 'POST'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
+ done();
+ });
+});
+
+test('SHP format, unauthenticated, with custom filename', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape',
+ headers: {host: 'vizzuality.cartodb.com'},
+ encoding: 'binary',
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=myshape.zip/gi.test(cd));
+ var tmpfile = '/tmp/myshape.zip';
+ var err = fs.writeFileSync(tmpfile, res.body, 'binary');
+ if (err) { done(err); return }
+ var zf = new zipfile.ZipFile(tmpfile);
+ assert.ok(_.contains(zf.names, 'myshape.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+ assert.ok(_.contains(zf.names, 'myshape.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+ assert.ok(_.contains(zf.names, 'myshape.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+ // missing SRID, so no PRJ (TODO: add ?)
+ //assert.ok(_.contains(zf.names, 'myshape.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+ fs.unlinkSync(tmpfile);
+ done();
+ });
+});
+
+test('SHP format, unauthenticated, with custom, dangerous filename', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=b;"%20()[]a',
+ headers: {host: 'vizzuality.cartodb.com'},
+ encoding: 'binary',
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var fname = "b_______a";
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=b_______a.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
+ var tmpfile = '/tmp/myshape.zip';
+ var err = fs.writeFileSync(tmpfile, res.body, 'binary');
+ if (err) { done(err); return }
+ var zf = new zipfile.ZipFile(tmpfile);
+ assert.ok(_.contains(zf.names, fname + '.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+ assert.ok(_.contains(zf.names, fname + '.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+ assert.ok(_.contains(zf.names, fname + '.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+ // missing SRID, so no PRJ (TODO: add ?)
+ //assert.ok(_.contains(zf.names, fname+ '.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+ fs.unlinkSync(tmpfile);
+ done();
+ });
+});
+
+test('SHP format, authenticated', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&api_key=1234',
+ headers: {host: 'vizzuality.cartodb.com'},
+ encoding: 'binary',
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
+ var tmpfile = '/tmp/myshape.zip';
+ var err = fs.writeFileSync(tmpfile, res.body, 'binary');
+ if (err) { done(err); return }
+ var zf = new zipfile.ZipFile(tmpfile);
+ assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+ assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+ assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+ // missing SRID, so no PRJ (TODO: add ?)
+ //assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+ // TODO: check contents of the DBF
+ fs.unlinkSync(tmpfile);
+ done();
+ });
+});
+
+// KML tests
+
+test('KML format, unauthenticated', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
- assert.equal(true, /filename=cartodb-query.json/gi.test(cd));
+ assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+ var row0 = res.body;
+ var checkfields = {'Name':1, 'address':1, 'cartodb_id':1, 'the_geom':0, 'the_geom_webmercator':0};
+ for ( var f in checkfields ) {
+ if ( checkfields[f] ) {
+ assert.ok(row0.indexOf('SimpleData name="'+ f + '"') != -1, "result does not include '" + f + "'");
+ } else {
+ assert.ok(row0.indexOf('SimpleData name="'+ f + '"') == -1, "result includes '" + f + "'");
+ }
+ }
done();
});
});
-test('GET /api/v1/sql ensure cross domain set on errors', function(done){
+test('KML format, unauthenticated, POST', function(done){
assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20*gadfgadfg%20FROM%20untitle_table_4',
- headers: {host: 'vizzuality.cartodb.com'},
- method: 'GET'
- },{
- status: 400
- }, function(res){
- var cd = res.header('Access-Control-Allow-Origin');
- assert.equal(cd, '*');
- done();
- });
-});
-
-test('GET /api/v1/sql as geojson limiting decimal places', function(done){
- assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&dp=1',
- headers: {host: 'vizzuality.cartodb.com'},
- method: 'GET'
+ url: '/api/v1/sql',
+ data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml',
+ headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+ method: 'POST'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
- var result = JSON.parse(res.body);
- assert.equal(1, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
done();
});
});
-test('GET /api/v1/sql as geojson with default dp as 6', function(done){
+test('KML format, skipfields', function(done){
assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
- headers: {host: 'vizzuality.cartodb.com'},
- method: 'GET'
- },{ }, function(res){
- assert.equal(res.statusCode, 200, res.body);
- var result = JSON.parse(res.body);
- assert.equal(6, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
- done();
- });
-});
-
-test('GET /api/v1/sql as csv', function(done){
- assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20cartodb_id,ST_AsEWKT(the_geom)%20as%20geom%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
- headers: {host: 'vizzuality.cartodb.com'},
- method: 'GET'
- },{ }, function(res){
- assert.equal(res.statusCode, 200, res.body);
- var body = "cartodb_id,geom\r\n1,SRID=4326;POINT(-3.699732 40.423012)";
- assert.equal(body, res.body);
- done();
- });
-});
-
-test('GET /api/v1/sql as csv, properly escaped', function(done){
- assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20cartodb_id,%20address%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
- headers: {host: 'vizzuality.cartodb.com'},
- method: 'GET'
- },{ }, function(res){
- assert.equal(res.statusCode, 200, res.body);
- var body = 'cartodb_id,address\r\n1,"Calle de Pérez Galdós 9, Madrid, Spain"';
- assert.equal(body, res.body);
- done();
- });
-});
-
-test('cannot GET system tables', function(done){
- assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20*%20FROM%20pg_attribute',
- headers: {host: 'vizzuality.cartodb.com'},
- method: 'GET'
- },{
- status: 403
- }, function() { done(); });
-});
-
-test('GET decent error if domain is incorrect', function(done){
- assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
- headers: {host: 'vizzualinot.cartodb.com'},
- method: 'GET'
- },{
- status: 404
- }, function(res){
- var result = JSON.parse(res.body);
- assert.equal(result.error[0],"Sorry, we can't find this CartoDB. Please check that you have entered the correct domain.");
- done();
- });
-});
-
-test('GET decent error if SQL is broken', function(done){
- assert.response(app, {
- url: '/api/v1/sql?' + querystring.stringify({q:
- 'SELECT star FROM this and that'
- }),
- headers: {host: 'vizzuality.cartodb.com'},
- method: 'GET'
- },{}, function(res){
- assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
- var result = JSON.parse(res.body);
- // NOTE: actual error message may be slighly different, possibly worth a regexp here
- assert.equal(result.error[0], 'syntax error at or near "and"');
- done();
- });
-});
-
-// CSV tests
-test('CSV format', function(done){
- assert.response(app, {
- url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
- assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
+ assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+ var row0 = res.body;
+ var checkfields = {'Name':1, 'address':0, 'cartodb_id':0, 'the_geom':0, 'the_geom_webmercator':0};
+ for ( var f in checkfields ) {
+ if ( checkfields[f] ) {
+ assert.ok(row0.indexOf('SimpleData name="'+ f + '"') != -1, "result does not include '" + f + "'");
+ } else {
+ assert.ok(row0.indexOf('SimpleData name="'+ f + '"') == -1, "result includes '" + f + "'");
+ }
+ }
done();
});
});
+test('KML format, unauthenticated, custom filename', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&filename=kmltest',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+ assert.equal(true, /filename=kmltest.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+ // TODO: check for actual content, at least try to uncompress..
+ done();
+ });
+});
+
+test('KML format, authenticated', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&api_key=1234',
+ headers: {host: 'vizzuality.cartodb.com'},
+ method: 'GET'
+ },{ }, function(res){
+ assert.equal(res.statusCode, 200, res.body);
+ var cd = res.header('Content-Disposition');
+ assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+ // TODO: check for actual content, at least try to uncompress..
+ done();
+ });
+});
+
+/**
+ * CORS
+ */
+test('GET /api/v1/sql with SQL parameter on SELECT only should return CORS headers ', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db',
+ method: 'GET'
+ },{ }, function(res) {
+ assert.equal(res.statusCode, 200, res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:untitle_table_4');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ assert.equal(res.headers['access-control-allow-origin'], '*');
+ assert.equal(res.headers['access-control-allow-headers'], "X-Requested-With, X-Prototype-Version, X-CSRF-Token");
+ done();
+ });
+});
+
+test('OPTIONS /api/v1/sql with SQL parameter on SELECT only should return CORS headers ', function(done){
+ assert.response(app, {
+ url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db',
+ method: 'OPTIONS'
+ },{ }, function(res) {
+ assert.equal(res.statusCode, 200, res.body);
+ // Check cache headers
+ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+ assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:untitle_table_4');
+ assert.equal(res.headers['cache-control'], expected_cache_control);
+ assert.equal(res.headers['access-control-allow-origin'], '*');
+ assert.equal(res.headers['access-control-allow-headers'], "X-Requested-With, X-Prototype-Version, X-CSRF-Token");
+ done();
+ });
+});
+
+
});
diff --git a/test/prepare_db.sh b/test/prepare_db.sh
index 909d7e95..b271dd01 100755
--- a/test/prepare_db.sh
+++ b/test/prepare_db.sh
@@ -34,6 +34,8 @@ echo "preparing postgres..."
dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise
createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database"
psql -f test.sql ${TEST_DB}
+psql -f support/CDB_QueryStatements.sql ${TEST_DB}
+psql -f support/CDB_QueryTables.sql ${TEST_DB}
echo "preparing redis..."
echo "HSET rails:users:vizzuality id 1" | redis-cli -p ${REDIS_PORT} -n 5
diff --git a/test/support/CDB_QueryStatements.sql b/test/support/CDB_QueryStatements.sql
new file mode 100644
index 00000000..9aecc18b
--- /dev/null
+++ b/test/support/CDB_QueryStatements.sql
@@ -0,0 +1,13 @@
+-- Return an array of statements found in the given query text
+--
+-- Curtesy of Hubert Lubaczewski (depesz)
+--
+CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
+RETURNS SETOF TEXT AS $$
+ SELECT stmt FROM (
+ SELECT btrim(q[1], E' \n\t\r;') as stmt FROM (
+ SELECT regexp_matches( $1, $REG$((?:[^'"$;]+|"[^"]*"|'(?:[^']*|'')*'|(\$[^$]*\$).*?\2)+)$REG$, 'g' ) as q
+ ) i
+ ) j
+ WHERE stmt <> '';
+$$ language sql;
diff --git a/test/support/CDB_QueryTables.sql b/test/support/CDB_QueryTables.sql
new file mode 100644
index 00000000..e504c293
--- /dev/null
+++ b/test/support/CDB_QueryTables.sql
@@ -0,0 +1,54 @@
+-- Return an array of table names scanned by a given query
+--
+-- Requires PostgreSQL 9.x+
+--
+CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
+RETURNS name[]
+AS $$
+DECLARE
+ exp XML;
+ tables NAME[];
+ rec RECORD;
+ rec2 RECORD;
+BEGIN
+
+ tables := '{}';
+
+ FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
+
+ BEGIN
+ EXECUTE 'EXPLAIN (FORMAT XML) ' || rec.q INTO STRICT exp;
+ EXCEPTION WHEN others THEN
+ RAISE WARNING 'Cannot explain query: % (%)', rec.q, SQLERRM;
+ CONTINUE;
+ END;
+
+ -- Now need to extract all values of
+
+ --RAISE DEBUG 'Explain: %', exp;
+
+ FOR rec2 IN WITH
+ inp AS ( SELECT xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x )
+ SELECT unnest(x)::name as p from inp
+ LOOP
+ --RAISE DEBUG 'tab: %', rec2.p;
+ tables := array_append(tables, rec2.p);
+ END LOOP;
+
+ -- RAISE DEBUG 'Tables: %', tables;
+
+ END LOOP;
+
+ -- RAISE DEBUG 'Tables: %', tables;
+
+ -- Remove duplicates and sort by name
+ IF array_upper(tables, 1) > 0 THEN
+ WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
+ SELECT array_agg(p) from dist into tables;
+ END IF;
+
+ --RAISE DEBUG 'Tables: %', tables;
+
+ return tables;
+END
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
diff --git a/test/test.sql b/test/test.sql
index c9db9360..11ad0c1f 100644
--- a/test/test.sql
+++ b/test/test.sql
@@ -20,31 +20,6 @@ SET search_path = public, pg_catalog;
SET default_tablespace = '';
SET default_with_oids = false;
-
--- Return an array of table names used by a given query
-CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
-RETURNS name[]
-AS $$
-DECLARE
- exp XML;
- tables NAME[];
-BEGIN
-
- EXECUTE 'EXPLAIN (FORMAT XML) ' || query INTO STRICT exp;
-
- -- Now need to extract all values of
-
- --RAISE DEBUG 'Explain: %', exp;
-
- tables := xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]);
-
- --RAISE DEBUG 'Tables: %', tables;
-
- return tables;
-END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
-
-
-- first table
DROP TABLE IF EXISTS untitle_table_4;
CREATE TABLE untitle_table_4 (
@@ -133,6 +108,7 @@ CREATE INDEX test_table_the_geom_webmercator_idx_p ON private_table USING gist (
CREATE USER publicuser WITH PASSWORD '';
CREATE USER test_cartodb_user_1 WITH PASSWORD '';
+GRANT ALL ON TABLE untitle_table_4 TO test_cartodb_user_1;
GRANT SELECT ON TABLE untitle_table_4 TO publicuser;
GRANT ALL ON TABLE private_table TO test_cartodb_user_1;
GRANT ALL ON SEQUENCE test_table_cartodb_id_seq_p TO test_cartodb_user_1