cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/v2.0'

Browse Source
This commit is contained in:
Luis Bosque 2012-11-30 17:01:44 +01:00
commit ae9298e04e
5 changed files with 127 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
NEWS.md
View File

@ -1,3 +1,8 @@
1.3.2 (30/11/12)
-----
* Fix KML export truncation (#70)
* Fix UTF8 in shapefile export (#66)
1.3.1 (DD/MM/YY)
-----
* Support 'format' and 'filename' params in POST

6
app/controllers/app.js
View File

@ -471,6 +471,7 @@ function toOGR(dbname, user_id, gcol, sql, skipfields, res, out_format, out_file
var child = spawn(ogr2ogr, [
'-f', out_format,
'-lco', 'ENCODING=UTF-8',
out_filename,
"PG:host=" + dbhost
+ " user=" + dbuser
@ -563,6 +564,7 @@ function toSHP(dbname, user_id, gcol, sql, skipfields, filename, res, callback)
var child = spawn(zip, ['-qrj', '-', dir ]);
// TODO: convert to a stream operation
child.stdout.on('data', function(data) {
res.write(data);
});
@ -663,8 +665,7 @@ function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) {
function sendResults(err) {
if ( ! err ) {
var stream = fs.createReadStream(dumpfile);
util.pump(stream, res);
fs.createReadStream(dumpfile).pipe(res);
}
// cleanup output dir (should be safe to unlink)
@ -706,7 +707,6 @@ function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) {
function finish(err) {
if ( err ) callback(err);
else {
res.end();
callback(null);
}

2
package.json
View File

@ -2,7 +2,7 @@
"private": true,
"name": "cartodb_api",
"description": "high speed SQL api for cartodb",
"version": "1.3.1",
"version": "1.3.2",
"author": {
"name": "Simon Tokumine, Sandro Santilli, Vizzuality",
"url": "http://vizzuality.com",

81
test/acceptance/app.test.js
View File

@ -726,6 +726,22 @@ test('CSV format', function(done){
});
});
test('CSV format, bigger than 81920 bytes', function(done){
assert.response(app, {
url: '/api/v1/sql',
data: querystring.stringify({
q: 'SELECT 0 as fname FROM generate_series(0,81920)',
format: 'csv'
}),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
assert.ok(res.body.length > 81920, 'CSV smaller than expected: ' + res.body.length);
done();
});
});
test('CSV format from POST', function(done){
assert.response(app, {
url: '/api/v1/sql',
@ -996,6 +1012,25 @@ test('SHP format, unauthenticated, POST', function(done){
});
});
test('SHP format, big size, POST', function(done){
assert.response(app, {
url: '/api/v1/sql',
data: querystring.stringify({
q: 'SELECT 0 as fname FROM generate_series(0,81920)',
format: 'shp'
}),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
assert.ok(res.body.length > 81920, 'SHP smaller than expected: ' + res.body.length);
done();
});
});
test('SHP format, unauthenticated, with custom filename', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape',
@ -1072,6 +1107,33 @@ test('SHP format, authenticated', function(done){
});
});
// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/66
test('SHP format, unauthenticated, with utf8 data', function(done){
var query = querystring.stringify({
q: "SELECT '♥♦♣♠' as f, st_makepoint(0,0,4326) as the_geom",
format: 'shp',
filename: 'myshape'
});
assert.response(app, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) { done(err); return }
var zf = new zipfile.ZipFile(tmpfile);
var buffer = zf.readFileSync('myshape.dbf');
fs.unlinkSync(tmpfile);
var strings = buffer.toString();
assert.ok(/♥♦♣♠/.exec(strings), "Cannot find '♥♦♣♠' in here:\n" + strings);
done();
});
});
// KML tests
test('KML format, unauthenticated', function(done){
@ -1112,6 +1174,25 @@ test('KML format, unauthenticated, POST', function(done){
});
});
test('KML format, bigger than 81920 bytes', function(done){
assert.response(app, {
url: '/api/v1/sql',
data: querystring.stringify({
q: 'SELECT 0 as fname FROM generate_series(0,81920)',
format: 'kml'
}),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
assert.ok(res.body.length > 81920, 'KML smaller than expected: ' + res.body.length);
done();
});
});
test('KML format, skipfields', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id',

37
tools/cdbsql
View File

@ -14,6 +14,7 @@ var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8;
//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support');
var readline = hasReadline ? require('readline') : null;
var tty = require('tty');
var me = process.argv[1];
@ -32,6 +33,7 @@ function usage(exit_code) {
console.log(" --key <string> API authentication key (none)");
console.log(" --format <string> Response format (json)");
console.log(" --dp <num> Decimal places in geojson format (unspecified)");
console.log(" --echo-queries echo commands sent to server");
if ( hasReadline ) {
console.log(" --batch Send all read queries at once (off)");
}
@ -50,6 +52,7 @@ var api_version = 1;
var api_key;
var sql;
var decimal_places;
var echo_queries = false;
var arg;
while ( arg = process.argv.shift() ) {
@ -83,6 +86,9 @@ while ( arg = process.argv.shift() ) {
else if ( arg == '--batch' ) {
batch_mode = true;
}
else if ( arg == '--echo-queries' ) {
echo_queries = true;
}
else if ( ! sql ) {
sql = arg;
}
@ -93,23 +99,27 @@ while ( arg = process.argv.shift() ) {
var hostname = username + '.' + domain;
if ( ! tty.isatty(process.stdin) || ! tty.isatty(process.stdout) ) {
batch_mode = true;
}
if ( ! sql ) {
if ( ! batch_mode ) {
if ( readline ) {
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
output: process.stdout,
});
if ( ! batch_mode ) {
rl.setPrompt(hostname + '> ');
rl.prompt();
}
sql = '';
rl.on('line', function(line) {
sql += line;
if ( ! batch_mode ) {
// TODO: some sanity checking, like trim the line or check if it ends with semicolon
if ( sql.length ) {
processQuery(sql, function() {
@ -117,19 +127,11 @@ if ( ! sql ) {
rl.prompt();
});
} else rl.prompt();
}
}).on('close', function() {
if ( batch_mode ) {
if ( sql.length ) {
processQuery(sql);
sql = '';
}
} else {
if ( sql.length ) {
console.warn("Unprocessed sql left: [" + sql + "]");
}
console.log("Good bye");
}
}).on('SIGCONT', function() {
// this is needed so not to exit on stop/resume
rl.prompt();
@ -137,6 +139,15 @@ if ( ! sql ) {
} else {
usage(1);
}
} else { // batch mode
process.stdin.resume();
process.stdin.setEncoding('utf8');
sql = '';
process.stdin.on('data', function(chunk) {
// TODO: some sanity checking, like trim the line or check if it ends with semicolon
processQuery(chunk);
});
}
} else {
processQuery(sql);
}
@ -177,7 +188,7 @@ function processQuery(sql, callback)
console.log("Request:", request);
var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql;
sqlprint = sqlprint.split('\n').join(' ');
console.log("Query:", sqlprint);
if ( echo_queries ) console.log("Query:", sqlprint);
console.log("Response status: " + res.statusCode);
console.log('Response body:');
console.dir(body);