diff --git a/NEWS.md b/NEWS.md index 60d1c33c..afda7cac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +1.3.2 (30/11/12) +----- +* Fix KML export truncation (#70) +* Fix UTF8 in shapefile export (#66) + 1.3.1 (DD/MM/YY) ----- * Support 'format' and 'filename' params in POST diff --git a/app/controllers/app.js b/app/controllers/app.js index 21599e5f..cbe43151 100755 --- a/app/controllers/app.js +++ b/app/controllers/app.js @@ -471,6 +471,7 @@ function toOGR(dbname, user_id, gcol, sql, skipfields, res, out_format, out_file var child = spawn(ogr2ogr, [ '-f', out_format, + '-lco', 'ENCODING=UTF-8', out_filename, "PG:host=" + dbhost + " user=" + dbuser @@ -563,6 +564,7 @@ function toSHP(dbname, user_id, gcol, sql, skipfields, filename, res, callback) var child = spawn(zip, ['-qrj', '-', dir ]); + // TODO: convert to a stream operation child.stdout.on('data', function(data) { res.write(data); }); @@ -663,8 +665,7 @@ function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) { function sendResults(err) { if ( ! err ) { - var stream = fs.createReadStream(dumpfile); - util.pump(stream, res); + fs.createReadStream(dumpfile).pipe(res); } // cleanup output dir (should be safe to unlink) @@ -706,7 +707,6 @@ function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) { function finish(err) { if ( err ) callback(err); else { - res.end(); callback(null); } diff --git a/package.json b/package.json index 4dcfc82b..f46b4eac 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "private": true, "name": "cartodb_api", "description": "high speed SQL api for cartodb", - "version": "1.3.1", + "version": "1.3.2", "author": { "name": "Simon Tokumine, Sandro Santilli, Vizzuality", "url": "http://vizzuality.com", diff --git a/test/acceptance/app.test.js b/test/acceptance/app.test.js index 08f8b578..15cf92a2 100644 --- a/test/acceptance/app.test.js +++ b/test/acceptance/app.test.js @@ -726,6 +726,22 @@ test('CSV format', function(done){ }); }); +test('CSV format, bigger than 81920 bytes', function(done){ + assert.response(app, { + url: '/api/v1/sql', + data: querystring.stringify({ + q: 'SELECT 0 as fname FROM generate_series(0,81920)', + format: 'csv' + }), + headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST' + },{ }, function(res){ + assert.ok(res.body.length > 81920, 'CSV smaller than expected: ' + res.body.length); + done(); + }); +}); + + test('CSV format from POST', function(done){ assert.response(app, { url: '/api/v1/sql', @@ -996,6 +1012,25 @@ test('SHP format, unauthenticated, POST', function(done){ }); }); +test('SHP format, big size, POST', function(done){ + assert.response(app, { + url: '/api/v1/sql', + data: querystring.stringify({ + q: 'SELECT 0 as fname FROM generate_series(0,81920)', + format: 'shp' + }), + headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd); + assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd); + assert.ok(res.body.length > 81920, 'SHP smaller than expected: ' + res.body.length); + done(); + }); +}); + test('SHP format, unauthenticated, with custom filename', function(done){ assert.response(app, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape', @@ -1072,6 +1107,33 @@ test('SHP format, authenticated', function(done){ }); }); + +// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/66 +test('SHP format, unauthenticated, with utf8 data', function(done){ + var query = querystring.stringify({ + q: "SELECT '♥♦♣♠' as f, st_makepoint(0,0,4326) as the_geom", + format: 'shp', + filename: 'myshape' + }); + assert.response(app, { + url: '/api/v1/sql?' + query, + headers: {host: 'vizzuality.cartodb.com'}, + encoding: 'binary', + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var tmpfile = '/tmp/myshape.zip'; + var err = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (err) { done(err); return } + var zf = new zipfile.ZipFile(tmpfile); + var buffer = zf.readFileSync('myshape.dbf'); + fs.unlinkSync(tmpfile); + var strings = buffer.toString(); + assert.ok(/♥♦♣♠/.exec(strings), "Cannot find '♥♦♣♠' in here:\n" + strings); + done(); + }); +}); + // KML tests test('KML format, unauthenticated', function(done){ @@ -1112,6 +1174,25 @@ test('KML format, unauthenticated, POST', function(done){ }); }); +test('KML format, bigger than 81920 bytes', function(done){ + assert.response(app, { + url: '/api/v1/sql', + data: querystring.stringify({ + q: 'SELECT 0 as fname FROM generate_series(0,81920)', + format: 'kml' + }), + headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd); + assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd); + assert.ok(res.body.length > 81920, 'KML smaller than expected: ' + res.body.length); + done(); + }); +}); + test('KML format, skipfields', function(done){ assert.response(app, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id', diff --git a/tools/cdbsql b/tools/cdbsql index cd54bfcd..6ad44b3e 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -14,6 +14,7 @@ var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8; //console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support'); var readline = hasReadline ? require('readline') : null; +var tty = require('tty'); var me = process.argv[1]; @@ -32,6 +33,7 @@ function usage(exit_code) { console.log(" --key API authentication key (none)"); console.log(" --format Response format (json)"); console.log(" --dp Decimal places in geojson format (unspecified)"); + console.log(" --echo-queries echo commands sent to server"); if ( hasReadline ) { console.log(" --batch Send all read queries at once (off)"); } @@ -50,6 +52,7 @@ var api_version = 1; var api_key; var sql; var decimal_places; +var echo_queries = false; var arg; while ( arg = process.argv.shift() ) { @@ -83,6 +86,9 @@ while ( arg = process.argv.shift() ) { else if ( arg == '--batch' ) { batch_mode = true; } + else if ( arg == '--echo-queries' ) { + echo_queries = true; + } else if ( ! sql ) { sql = arg; } @@ -93,23 +99,27 @@ while ( arg = process.argv.shift() ) { var hostname = username + '.' + domain; +if ( ! tty.isatty(process.stdin) || ! tty.isatty(process.stdout) ) { + batch_mode = true; +} + if ( ! sql ) { - if ( readline ) { - var rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }); + if ( ! batch_mode ) { + + if ( readline ) { + + var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); - if ( ! batch_mode ) { rl.setPrompt(hostname + '> '); rl.prompt(); - } - sql = ''; - rl.on('line', function(line) { - sql += line; - if ( ! batch_mode ) { + sql = ''; + rl.on('line', function(line) { + sql += line; // TODO: some sanity checking, like trim the line or check if it ends with semicolon if ( sql.length ) { processQuery(sql, function() { @@ -117,25 +127,26 @@ if ( ! sql ) { rl.prompt(); }); } else rl.prompt(); - } - }).on('close', function() { - if ( batch_mode ) { - if ( sql.length ) { - processQuery(sql); - sql = ''; - } - } else { + }).on('close', function() { if ( sql.length ) { console.warn("Unprocessed sql left: [" + sql + "]"); } console.log("Good bye"); - } - }).on('SIGCONT', function() { - // this is needed so not to exit on stop/resume - rl.prompt(); + }).on('SIGCONT', function() { + // this is needed so not to exit on stop/resume + rl.prompt(); + }); + } else { + usage(1); + } + } else { // batch mode + process.stdin.resume(); + process.stdin.setEncoding('utf8'); + sql = ''; + process.stdin.on('data', function(chunk) { + // TODO: some sanity checking, like trim the line or check if it ends with semicolon + processQuery(chunk); }); - } else { - usage(1); } } else { processQuery(sql); @@ -177,7 +188,7 @@ function processQuery(sql, callback) console.log("Request:", request); var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql; sqlprint = sqlprint.split('\n').join(' '); - console.log("Query:", sqlprint); + if ( echo_queries ) console.log("Query:", sqlprint); console.log("Response status: " + res.statusCode); console.log('Response body:'); console.dir(body);