cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for specifying a filename for exports. Closes #64

Browse Source 
Sets release target to 1.3.0, due to parameter addition
This commit is contained in:
Sandro Santilli 2012-11-12 12:37:34 +01:00
parent 755fe738ca
commit 46cec7a0e5
5 changed files with 171 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
NEWS.md
View File

@ -1,5 +1,6 @@
1.2.2 (DD/MM/YY)
1.3.0 (DD/MM/YY)
-----
* Support for specifying a filename for exports (#64)
1.2.1 (DD/MM/YY)
-----

24
app/controllers/app.js
View File

@ -14,6 +14,9 @@
// eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table
//
//
var path = require('path');
var express = require('express')
, app = express.createServer(
express.logger({
@ -65,6 +68,12 @@ function userid_to_dbuser(user_id) {
return "publicuser" // FIXME: make configurable
};
function sanitize_filename(filename) {
filename = path.basename(filename, path.extname(filename));
filename = filename.replace(/[;()\[\]<>'"\s]/g, '_');
//console.log("Sanitized: " + filename);
return filename;
}
// request handlers
function handleQuery(req, res) {
@ -81,6 +90,7 @@ function handleQuery(req, res) {
var limit = parseInt(req.query.rows_per_page);
var offset = parseInt(req.query.page);
var format = _.isArray(req.query.format) ? _.last(req.query.format) : req.query.format;
var filename = req.query.filename;
var dp = req.query.dp; // decimal point digits (defaults to 6)
var gn = "the_geom"; // TODO: read from configuration file
var user_id;
@ -88,12 +98,12 @@ function handleQuery(req, res) {
// sanitize and apply defaults to input
dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp;
format = (format === "" || _.isUndefined(format)) ? 'json' : format.toLowerCase();
filename = (filename === "" || _.isUndefined(filename)) ? 'cartodb-query' : sanitize_filename(filename);
sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
database = (database === "" || _.isUndefined(database)) ? null : database;
limit = (_.isNumber(limit)) ? limit : null;
offset = (_.isNumber(offset)) ? offset * limit : null;
// setup step run
var start = new Date().getTime();
@ -205,7 +215,7 @@ function handleQuery(req, res) {
if (err) throw err;
// configure headers for given format
res.header("Content-Disposition", getContentDisposition(format));
res.header("Content-Disposition", getContentDisposition(format, filename));
res.header("Content-Type", getContentType(format));
// allow cross site post
@ -229,7 +239,7 @@ function handleQuery(req, res) {
} else if (format === 'csv'){
toCSV(result, res, this);
} else if ( format === 'shp'){
toSHP(database, user_id, gn, sql, res, this);
toSHP(database, user_id, gn, sql, filename, res, this);
} else if ( format === 'kml'){
toKML(database, user_id, gn, sql, res, this);
} else if ( format === 'json'){
@ -489,11 +499,11 @@ console.log(['ogr2ogr',
);
}
function toSHP(dbname, user_id, gcol, sql, res, callback) {
function toSHP(dbname, user_id, gcol, sql, filename, res, callback) {
var zip = 'zip'; // FIXME: make configurable
var tmpdir = '/tmp'; // FIXME: make configurable
var outdirpath = tmpdir + '/sqlapi-shapefile-' + generateMD5(sql);
var shapefile = outdirpath + '/cartodb-query.shp';
var shapefile = outdirpath + '/' + filename + '.shp';
// TODO: following tests:
// - fetch with no auth [done]
@ -683,7 +693,7 @@ function toKML(dbname, user_id, gcol, sql, res, callback) {
);
}
function getContentDisposition(format){
function getContentDisposition(format, filename) {
var ext = 'json';
if (format === 'geojson'){
ext = 'geojson';
@ -701,7 +711,7 @@ function getContentDisposition(format){
ext = 'kml';
}
var time = new Date().toUTCString();
return 'attachment; filename=cartodb-query.' + ext + '; modification-date="' + time + '";';
return 'attachment; filename=' + filename + '.' + ext + '; modification-date="' + time + '";';
}
function getContentType(format){

7
npm-shrinkwrap.json generated
View File

@ -1,9 +1,9 @@
{
"name": "cartodb_api",
"version": "1.1.0",
"version": "1.3.0-dev",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb01",
"version": "0.3.5-cdb02",
"from": "git://github.com/CartoDB/cluster2.git#cdb_production",
"dependencies": {
"ejs": {
@ -218,6 +218,9 @@
"csv": {
"version": "0.0.13"
},
"zipfile": {
"version": "0.3.2"
},
"mocha": {
"version": "1.2.1",
"dependencies": {

5
package.json
View File

@ -2,7 +2,7 @@
"private": true,
"name": "cartodb_api",
"description": "high speed SQL api for cartodb",
"version": "1.2.2",
"version": "1.3.0-dev",
"author": {
"name": "Simon Tokumine, Sandro Santilli, Vizzuality",
"url": "http://vizzuality.com",
@ -23,7 +23,8 @@
"csv":"0.0.13"
},
"devDependencies": {
"mocha": "1.2.1"
"mocha": "1.2.1",
"zipfile": "~0.3.2"
},
"scripts": {
"test": "test/run_tests.sh"

148
test/acceptance/app.test.js
View File

@ -14,10 +14,14 @@
require('../helper');
require('../support/assert');
var app = require(global.settings.app_root + '/app/controllers/app')
, assert = require('assert')
, querystring = require('querystring')
, _ = require('underscore');
, _ = require('underscore')
, zipfile = require('zipfile')
, fs = require('fs')
;
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
@ -422,7 +426,7 @@ test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposi
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'JSON is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.json/gi.test(cd));
assert.equal(true, /filename=cartodb-query.json/gi.test(cd), 'Unexpected JSON filename: ' + cd);
done();
});
});
@ -510,6 +514,19 @@ test('uses the last format parameter when multiple are used', function(done){
});
});
test('uses custom filename', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&filename=x',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /filename=x.geojson/gi.test(cd), cd);
done();
});
});
test('GET /api/v1/sql as geojson limiting decimal places', function(done){
assert.response(app, {
@ -561,6 +578,22 @@ test('CSV format', function(done){
});
});
test('CSV format, custom filename', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv&filename=mycsv.csv',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
assert.equal(true, /filename=mycsv.csv/gi.test(cd), cd);
var ct = res.header('Content-Type');
assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
done();
});
});
test('GET /api/v1/sql as csv', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20cartodb_id,ST_AsEWKT(the_geom)%20as%20geom%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
@ -623,6 +656,27 @@ test('GET /api/v1/sql with SVG format', function(done){
});
});
test('GET /api/v1/sql with SVG format and custom filename', function(done){
var query = querystring.stringify({
q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
format: "svg",
filename: 'mysvg'
});
assert.response(app, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.ok(/filename=mysvg.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
// TODO: test viewBox
done();
});
});
test('GET /api/v1/sql with SVG format and centered point', function(done){
var query = querystring.stringify({
q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ",
@ -687,13 +741,74 @@ test('SHP format, unauthenticated', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
// TODO: check for actual content, at least try to uncompress..
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) { done(err); return }
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
// missing SRID, so no PRJ (TODO: add ?)
//assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
fs.unlinkSync(tmpfile);
done();
});
});
test('SHP format, unauthenticated, with custom filename', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=myshape.zip/gi.test(cd));
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) { done(err); return }
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'myshape.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
assert.ok(_.contains(zf.names, 'myshape.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
assert.ok(_.contains(zf.names, 'myshape.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
// missing SRID, so no PRJ (TODO: add ?)
//assert.ok(_.contains(zf.names, 'myshape.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
fs.unlinkSync(tmpfile);
done();
});
});
test('SHP format, unauthenticated, with custom, dangerous filename', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=b;"%20()[]a',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var fname = "b_______a";
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=b_______a.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) { done(err); return }
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, fname + '.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
assert.ok(_.contains(zf.names, fname + '.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
assert.ok(_.contains(zf.names, fname + '.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
// missing SRID, so no PRJ (TODO: add ?)
//assert.ok(_.contains(zf.names, fname+ '.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
fs.unlinkSync(tmpfile);
done();
});
});
@ -702,12 +817,22 @@ test('SHP format, authenticated', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&api_key=1234',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
// TODO: check for actual content, at least try to uncompress..
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) { done(err); return }
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
// missing SRID, so no PRJ (TODO: add ?)
//assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
fs.unlinkSync(tmpfile);
done();
});
});
@ -729,6 +854,21 @@ test('KML format, unauthenticated', function(done){
});
});
test('KML format, unauthenticated, custom filename', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&filename=kmltest',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=kmltest.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
// TODO: check for actual content, at least try to uncompress..
done();
});
});
test('KML format, authenticated', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&api_key=1234',