Merge branch 'release/v2.0'

NEWS.md

@@ -1,3 +1,8 @@
+1.3.2 (30/11/12)
+-----
+* Fix KML export truncation (#70)
+* Fix UTF8 in shapefile export (#66)
+
 1.3.1 (DD/MM/YY)
 -----
 * Support 'format' and 'filename' params in POST

app/controllers/app.js

@@ -471,6 +471,7 @@ function toOGR(dbname, user_id, gcol, sql, skipfields, res, out_format, out_file
 
       var child = spawn(ogr2ogr, [
         '-f', out_format,
+        '-lco', 'ENCODING=UTF-8',
         out_filename,
         "PG:host=" + dbhost
          + " user=" + dbuser
@@ -563,6 +564,7 @@ function toSHP(dbname, user_id, gcol, sql, skipfields, filename, res, callback)
 
       var child = spawn(zip, ['-qrj', '-', dir ]);
 
+      // TODO: convert to a stream operation
       child.stdout.on('data', function(data) {
         res.write(data);
       });
@@ -663,8 +665,7 @@ function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) {
     function sendResults(err) {
 
       if ( ! err ) {
-        var stream = fs.createReadStream(dumpfile);
+        fs.createReadStream(dumpfile).pipe(res);
-        util.pump(stream, res);
       }
 
       // cleanup output dir (should be safe to unlink)
@@ -706,7 +707,6 @@ function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) {
     function finish(err) {
       if ( err ) callback(err);
       else {
-        res.end();
         callback(null);
       }
 

package.json

@@ -2,7 +2,7 @@
     "private": true,
     "name": "cartodb_api",
     "description": "high speed SQL api for cartodb",
-    "version": "1.3.1",
+    "version": "1.3.2",
     "author": {
         "name": "Simon Tokumine, Sandro Santilli, Vizzuality",
         "url": "http://vizzuality.com",

test/acceptance/app.test.js

@@ -726,6 +726,22 @@ test('CSV format', function(done){
     });
 });
 
+test('CSV format, bigger than 81920 bytes', function(done){
+    assert.response(app, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({
+          q: 'SELECT 0 as fname FROM generate_series(0,81920)',
+          format: 'csv'
+        }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(res){
+        assert.ok(res.body.length > 81920, 'CSV smaller than expected: ' + res.body.length);
+        done();
+    });
+});
+
+
 test('CSV format from POST', function(done){
     assert.response(app, {
         url: '/api/v1/sql',
@@ -996,6 +1012,25 @@ test('SHP format, unauthenticated, POST', function(done){
     });
 });
 
+test('SHP format, big size, POST', function(done){
+    assert.response(app, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({
+          q: 'SELECT 0 as fname FROM generate_series(0,81920)',
+          format: 'shp'
+        }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.header('Content-Disposition');
+        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
+        assert.ok(res.body.length > 81920, 'SHP smaller than expected: ' + res.body.length);
+        done();
+    });
+});
+
 test('SHP format, unauthenticated, with custom filename', function(done){
     assert.response(app, {
         url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape',
@@ -1072,6 +1107,33 @@ test('SHP format, authenticated', function(done){
     });
 });
 
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/66
+test('SHP format, unauthenticated, with utf8 data', function(done){
+    var query = querystring.stringify({
+        q: "SELECT '♥♦♣♠' as f, st_makepoint(0,0,4326) as the_geom",
+        format: 'shp',
+        filename: 'myshape'
+      });
+    assert.response(app, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        var tmpfile = '/tmp/myshape.zip';
+        var err = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (err) { done(err); return }
+        var zf = new zipfile.ZipFile(tmpfile);
+        var buffer = zf.readFileSync('myshape.dbf');
+        fs.unlinkSync(tmpfile);
+        var strings = buffer.toString();
+        assert.ok(/♥♦♣♠/.exec(strings), "Cannot find '♥♦♣♠' in here:\n" + strings);
+        done();
+    });
+});
+
 // KML tests
 
 test('KML format, unauthenticated', function(done){
@@ -1112,6 +1174,25 @@ test('KML format, unauthenticated, POST', function(done){
     });
 });
 
+test('KML format, bigger than 81920 bytes', function(done){
+    assert.response(app, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({
+          q: 'SELECT 0 as fname FROM generate_series(0,81920)',
+          format: 'kml'
+        }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.header('Content-Disposition');
+        assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        assert.ok(res.body.length > 81920, 'KML smaller than expected: ' + res.body.length);
+        done();
+    });
+});
+
 test('KML format, skipfields', function(done){
     assert.response(app, {
         url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id',

tools/cdbsql

@@ -14,6 +14,7 @@ var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8;
 //console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support');
 
 var readline = hasReadline ? require('readline') : null;
+var tty = require('tty');
 
 var me = process.argv[1];
 
@@ -32,6 +33,7 @@ function usage(exit_code) {
   console.log(" --key <string>          API authentication key (none)");
   console.log(" --format <string>       Response format (json)");
   console.log(" --dp <num>              Decimal places in geojson format (unspecified)");
+  console.log(" --echo-queries          echo commands sent to server");
   if ( hasReadline ) {
   console.log(" --batch                 Send all read queries at once (off)");
   }
@@ -50,6 +52,7 @@ var api_version = 1;
 var api_key;
 var sql;
 var decimal_places;
+var echo_queries = false;
 
 var arg;
 while ( arg = process.argv.shift() ) {
@@ -83,6 +86,9 @@ while ( arg = process.argv.shift() ) {
   else if ( arg == '--batch' ) {
     batch_mode = true;
   }
+  else if ( arg == '--echo-queries' ) {
+    echo_queries = true;
+  }
   else if ( ! sql ) {
     sql = arg;
   }
@@ -93,23 +99,27 @@ while ( arg = process.argv.shift() ) {
 
 var hostname = username + '.' + domain;
 
+if ( ! tty.isatty(process.stdin) || ! tty.isatty(process.stdout) ) {
+  batch_mode = true;
+}
+
 if ( ! sql ) {
-  if ( readline ) {
 
-    var rl = readline.createInterface({
+  if ( ! batch_mode ) {
-      input: process.stdin,
+
-      output: process.stdout
+    if ( readline ) {
-    });
+
+      var rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+      });
 
-    if ( ! batch_mode ) {
       rl.setPrompt(hostname + '> ');
       rl.prompt();
-    }
 
-    sql = '';
+      sql = '';
-    rl.on('line', function(line) {
+      rl.on('line', function(line) {
-      sql += line;
+        sql += line;
-      if ( ! batch_mode ) {
         // TODO: some sanity checking, like trim the line or check if it ends with semicolon
         if ( sql.length ) {
           processQuery(sql, function() {
@@ -117,25 +127,26 @@ if ( ! sql ) {
             rl.prompt();
           });
         } else rl.prompt();
-      } 
+      }).on('close', function() {
-    }).on('close', function() {
-      if ( batch_mode ) {
-        if ( sql.length )  {
-          processQuery(sql);
-          sql = '';
-        }
-      } else {
         if ( sql.length ) {
           console.warn("Unprocessed sql left: [" + sql + "]");
         }
         console.log("Good bye");
-      }
+      }).on('SIGCONT', function() {
-    }).on('SIGCONT', function() {
+        // this is needed so not to exit on stop/resume
-      // this is needed so not to exit on stop/resume
+        rl.prompt();
-      rl.prompt();
+      });
+    } else {
+      usage(1);
+    }
+  } else { // batch mode
+    process.stdin.resume();
+    process.stdin.setEncoding('utf8');
+    sql = '';
+    process.stdin.on('data', function(chunk) {
+      // TODO: some sanity checking, like trim the line or check if it ends with semicolon
+      processQuery(chunk);
     });
-  } else {
-    usage(1);
   }
 } else {
   processQuery(sql);
@@ -177,7 +188,7 @@ function processQuery(sql, callback)
       console.log("Request:", request);
       var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql;
       sqlprint = sqlprint.split('\n').join(' ');
-      console.log("Query:", sqlprint);
+      if ( echo_queries ) console.log("Query:", sqlprint);
       console.log("Response status: " + res.statusCode);
       console.log('Response body:');
       console.dir(body);