Merge branch 'release/staging'

Conflicts:
	NEWS.md
	package.json

.gitignore

@@ -1,6 +1,7 @@
+config/environments/*.js
 logs/*.log
 pids/*.pid
 *.sock
 test/tmp/*
 node_modules/
-.idea/*
+.idea/*

NEWS.md

@@ -1,3 +1,14 @@
+1.1.0 (30/10/12)
+-----
+* Fixed problem in cluster2 with pidfile name
+* SVG output format
+* Enhancement to the cdbsql tool:
+  - New switches: --format, --key, --dp
+  - Interactive mode
+* API documentation
+* ./configure script
+* Restrict listening to a node host
+
 1.0.0 (03/10/12)
 -----
 * Migrated to node 0.8 version

README.md

@@ -10,26 +10,11 @@ Provides a nodejs based API for running SQL queries against CartoDB.
 core requirements
 -------------
 * postgres 9.0+
-* cartodb 0.9.5+ (for CDB_QueryTables)
+* cartodb 0.9.5+ (for ``CDB_QueryTables``)
 * redis
-* node > v0.4.8 && < v0.9.0
+* node 0.8+
 * npm
 
-usage
------
-
-Edit config/environments/<environment>.js
-Make sure redis is running and knows about active cartodb user.
-
-``` bash
-node [cluster.js|app.js] <environment>
-```
-
-Supported <environment> values are developement, test, production
-
-for examples of use, see /tests
-
-
 Install dependencies
 ---------------------
 
@@ -37,12 +22,32 @@ Install dependencies
 npm install
 ```
 
+usage
+-----
+
+Create and edit config/environments/<environment>.js from .js.example files.
+You may find the ./configure script useful to make an edited copy for you,
+see ```./configure --help``` for a list of supported switches.
+
+Make sure redis is running and knows about active cartodb user.
+
+Make sure your PostgreSQL server is running, is accessible on
+the host and port specified in the <environment> file, has
+a 'publicuser' role and trusts user authentication from localhost
+connections.
+
+``` bash
+node [cluster.js|app.js] <environment>
+```
+
+Supported <environment> values are developement, test, production
+
+See doc/API.md for API documentation.
+For examples of use, see under test/.
+
 
 tests
 ------
-see test/README.md
 
+Run ```make check``` or see test/README.md
 
-note on 0.4.x
---------------
-output of large result sets is slow under node 0.4. Recommend running under 0.6 where possible.

app.js

@@ -26,5 +26,6 @@ _.extend(global.settings, env);
  
 // kick off controller
 var app = require(global.settings.app_root + '/app/controllers/app');
-app.listen(global.settings.node_port);
+app.listen(global.settings.node_port, global.settings.node_host, function() {
-console.log("CartoDB SQL API listening on port " + global.settings.node_port); 
+  console.log("CartoDB SQL API listening on " + global.settings.node_host + ":" + global.settings.node_port); 
+});

app/controllers/app.js

@@ -12,6 +12,8 @@
 // - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public
 //
 // eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table
+//
+//
 var express = require('express')
     , app      = express.createServer(
     express.logger({
@@ -47,7 +49,11 @@ function handleQuery(req, res) {
     var limit     = parseInt(req.query.rows_per_page);
     var offset    = parseInt(req.query.page);
     var format    = req.query.format;
-    var dp        = req.query.dp;
+    var dp        = req.query.dp; // decimal point digits (defaults to 6)
+    var gn        = "the_geom"; // TODO: read from configuration file 
+    var svg_width  = 1024.0;
+    var svg_height = 768.0;
+
 
     // sanitize and apply defaults to input
     dp        = (dp       === "" || _.isUndefined(dp))       ? '6'  : dp;
@@ -127,6 +133,27 @@ function handleQuery(req, res) {
                 // TODO: refactor formats to external object
                 if (format === 'geojson'){
                     sql = ['SELECT *, ST_AsGeoJSON(the_geom,',dp,') as the_geom FROM (', sql, ') as foo'].join("");
+                } else if (format === 'svg'){
+                    var svg_ratio = svg_width/svg_height;
+                    sql = 'WITH source AS ( ' + sql + '), extent AS ( ' 
+                        + ' SELECT ST_Extent(' + gn + ') AS e FROM source '
+                        + '), extent_info AS ( SELECT e, '
+                        + 'st_xmin(e) as ex0, st_ymax(e) as ey0, '
+                        + 'st_xmax(e)-st_xmin(e) as ew, '
+                        + 'st_ymax(e)-st_ymin(e) as eh FROM extent )'
+                        + ', trans AS ( SELECT CASE WHEN '
+                        + 'eh = 0 THEN ' + svg_width
+                        + '/ COALESCE(NULLIF(ew,0),' + svg_width +') WHEN '
+                        + svg_ratio + ' <= (ew / eh) THEN ('
+                        + svg_width  + '/ew ) ELSE ('
+                        + svg_height + '/eh ) END as s '
+                        + ', ex0 as x0, ey0 as y0 FROM extent_info ) '
+                        + 'SELECT st_TransScale(e, -x0, -y0, s, s)::box2d as '
+                        + gn + '_box, ST_Dimension(' + gn + ') as ' + gn
+                        + '_dimension, ST_AsSVG(ST_TransScale(' + gn + ', '
+                        + '-x0, -y0, s, s), 0, ' + dp + ') as ' + gn
+                        //+ ', ex0, ey0, ew, eh, s ' // DEBUG ONLY
+                        + ' FROM trans, extent_info, source';
                 }
 
                 pg.query(sql, this);
@@ -154,9 +181,12 @@ function handleQuery(req, res) {
                 // TODO: refactor formats to external object
                 if (format === 'geojson'){
                     toGeoJSON(result, res, this);
+                } else if (format === 'svg'){
+                    toSVG(result.rows, gn, this);
                 } else if (format === 'csv'){
                     toCSV(result, res, this);
                 } else {
+                    // TODO: error out if 'format' resolves to an unsupported format !
                     var end = new Date().getTime();
 
                     var json_result = {'time' : (end - start)/1000};
@@ -218,6 +248,98 @@ function toGeoJSON(data, res, callback){
     }
 }
 
+function toSVG(rows, gn, callback){
+
+    var radius = 5; // in pixels (based on svg_width and svg_height)
+    var stroke_width = 1; // in pixels (based on svg_width and svg_height)
+    var stroke_color = 'black';
+    // fill settings affect polygons and points (circles)
+    var fill_opacity = 0.5; // 0.0 is fully transparent, 1.0 is fully opaque
+                            // unused if fill_color='none'
+    var fill_color = 'none'; // affects polygons and circles
+
+    var bbox; // will be computed during the results scan
+    var polys = [];
+    var lines = [];
+    var points = [];
+    _.each(rows, function(ele){
+        var g = ele[gn];
+        if ( ! g ) return; // null or empty
+        var gdims = ele[gn + '_dimension'];
+
+        // TODO: add an identifier, if any of "cartodb_id", "oid", "id", "gid" are found
+        // TODO: add "class" attribute to help with styling ?
+        if ( gdims == '0' ) {
+          points.push('<circle r="[RADIUS]" ' + g + ' />');
+        } else if ( gdims == '1' ) {
+          // Avoid filling closed linestrings
+          var linetag = '<path ';
+          if ( fill_color != 'none' ) linetag += 'fill="none" '
+          linetag += 'd="' + g + '" />';
+          lines.push(linetag);
+        } else if ( gdims == '2' ) {
+          polys.push('<path d="' + g + '" />');
+        }
+
+        if ( ! bbox ) { 
+          // Parse layer extent: "BOX(x y, X Y)"
+          // NOTE: the name of the extent field is
+          //       determined by the same code adding the
+          //       ST_AsSVG call (in queryResult)
+          //
+          bbox = ele[gn + '_box'];
+          bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/);
+          bbox = {
+            xmin: parseFloat(bbox[1]),
+            ymin: parseFloat(bbox[2]), 
+            xmax: parseFloat(bbox[3]),
+            ymax: parseFloat(bbox[4]) 
+           };
+        }
+    });
+
+    // Set point radius
+    for (var i=0; i<points.length; ++i) {
+      points[i] = points[i].replace('[RADIUS]', radius);
+    }
+
+    var header_tags = [
+        '<?xml version="1.0" standalone="no"?>',
+        '<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">',
+    ];
+
+    var root_tag = '<svg ';
+    if ( bbox ) {
+      // expand box by "radius" + "stroke-width"
+      // TODO: use a Box2d class for these ops
+      var growby = radius+stroke_width;
+      bbox.xmin -= growby;
+      bbox.ymin -= growby;
+      bbox.xmax += growby;
+      bbox.ymax += growby;
+      bbox.width = bbox.xmax - bbox.xmin;
+      bbox.height = bbox.ymax - bbox.ymin;
+      root_tag += 'viewBox="' + bbox.xmin + ' ' + (-bbox.ymax) + ' '
+               + bbox.width + ' ' + bbox.height + '" '; 
+    }
+    root_tag += 'style="fill-opacity:' + fill_opacity
+              + '; stroke:' + stroke_color
+              + '; stroke-width:' + stroke_width
+              + '; fill:' + fill_color
+              + '" ';
+    root_tag += 'xmlns="http://www.w3.org/2000/svg" version="1.1">';
+
+    header_tags.push(root_tag);
+
+    // Render points on top of lines and lines on top of polys
+    var out = header_tags.concat(polys, lines, points);
+
+    out.push('</svg>');
+
+    // return payload
+    callback(null, out.join("\n"));
+}
+
 function toCSV(data, res, callback){
     try{
         // pull out keys for column headers
@@ -238,9 +360,12 @@ function getContentDisposition(format){
     if (format === 'geojson'){
         ext = 'geojson';
     }
-    if (format === 'csv'){
+    else if (format === 'csv'){
         ext = 'csv';
     }
+    else if (format === 'svg'){
+        ext = 'svg';
+    }
     var time = new Date().toUTCString();
     return 'inline; filename=cartodb-query.' + ext + '; modification-date="' + time + '";';
 }
@@ -250,6 +375,9 @@ function getContentType(format){
     if (format === 'csv'){
         type = "text/csv; charset=utf-8";
     }
+    else if (format === 'svg'){
+        type = "image/svg+xml; charset=utf-8";
+    }
     return type;
 }
 

cluster.js

@@ -30,11 +30,13 @@ var app = require(global.settings.app_root + '/app/controllers/app');
 
 var cluster = new Cluster({
   port: global.settings.node_port,
+  host: global.settings.node_host,
+  monHost: global.settings.node_host,
   monPort: global.settings.node_port+1
 });
 
 cluster.listen(function(cb) {
   cb(app);
+}, function() {
+  console.log("CartoDB SQL API listening on " + global.settings.node_host + ':' + global.settings.node_port); 
 });
-
-console.log("CartoDB SQL API listening on port " + global.settings.node_port); 

config/environments/development.js.example

@@ -1,4 +1,5 @@
 module.exports.node_port    = 8080;
+module.exports.node_host    = '127.0.0.1';
 module.exports.environment  = 'development';
 module.exports.db_base_name = 'cartodb_dev_user_<%= user_id %>_db';
 module.exports.db_user      = 'development_cartodb_user_<%= user_id %>';
@@ -9,4 +10,4 @@ module.exports.redis_port   = 6379;
 module.exports.redisPool    = 50;
 module.exports.redisIdleTimeoutMillis   = 100;
 module.exports.redisReapIntervalMillis  = 10;
-module.exports.redisLog     = false;
+module.exports.redisLog     = false;

configure

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+usage() {
+  echo "Usage: $0 [OPTION]"
+  echo 
+  echo "Configuration:"
+  echo "  --help                 display this help and exit"
+  echo "  --with-pgport=NUM      access PostgreSQL server on TCP port NUM"
+}
+
+PGPORT=5432
+
+while test -n "$1"; do
+  case "$1" in
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    --with-pgport=*)
+      PGPORT=`echo "$1" | cut -d= -f2`
+      ;;
+    *)
+      echo "Unknown option '$1'" >&2
+      usage >&2
+      exit 1
+  esac
+  shift
+done
+
+echo "PGPORT: $PGPORT"
+
+# TODO: allow specifying configuration settings !
+for f in config/environments/*.example; do
+  o=`dirname "$f"`/`basename "$f" .example`
+  echo "Writing $o"
+  sed "s/\( *module.exports.db_port[ \t]*= *'\?\)[^';]*\('\?;\)/\1$PGPORT\2/" < "$f" > "$o"
+done

doc/API.md

@@ -0,0 +1,127 @@
+SQL API
+=======
+
+Request format
+--------------
+
+Supported query string parameters:
+
+  'q':        Specifies the SQL query to run
+              Example:
+              'http://entrypoint?q=SELECT count(*) FROM mytable'
+
+  'format':   Specifies which format to use for the response.
+              Supported formats: JSON (the default), GeoJSON,
+              CSV, SVG
+
+  'dp':       Number of digits after the decimal point.
+              Only affects format=GeoJSON.
+              By default this is 6.
+
+  'api_key':  Needed to authenticate in order to modify the database.
+
+Response formats
+----------------
+
+The standard response from the CartoDB SQL API is JSON. If you are
+building a web-application, the lightweight JSON format allows you to
+quickly integrate data from the SQL API.
+
+The JSON response is as follows:
+```
+    {
+        time: 0.006,
+        total_rows: 1,
+        rows: [
+            {
+                year: "  2011",
+                the_geom: "0101000020E610...",
+                cartodb_id: 1,
+                created_at: "2012-02-06T22:50:35.778Z",
+                updated_at: "2012-02-12T21:34:08.193Z"
+            }
+        ]
+    }
+```
+
+Alternatively, you can use the GeoJSON specification for returning data
+from the API. To do so, simply supply the format parameter as GeoJSON.
+
+The GeoJSON response is follows:
+```
+    {
+        type: "FeatureCollection",
+        features: [
+            {
+                type: "Feature",
+                properties: {
+                    year: "  2011",
+                    month: 10,
+                    day: "11",
+                    cartodb_id: 1,
+                    created_at: "2012-02-06T22:50:35.778Z",
+                    updated_at: "2012-02-12T21:34:08.193Z"
+                },
+                geometry: {
+                    type: "Point",
+                    coordinates: [
+                        -97.335,
+                        35.498
+                    ]
+                }
+            }
+        ]
+    }
+```
+
+TODO: csv, kml responses
+
+Response errors
+---------------
+
+To help you debug your SQL queries, the CartoDB SQL API returns errors
+as part of the JSON response. Errors come back as follows,
+
+```
+    {
+        error: [
+          "syntax error at or near "LIMIT""
+        ]
+    }
+```
+
+You can use these errors to help understand your SQL.
+
+
+Getting table information
+-------------------------
+
+Currently, there is no public method for accessing your table schemas. The
+simplest way to get table structure is to access the first row of the data:
+
+    http://entrypoint?q=SELECT * FROM mytable LIMIT 1
+
+Write data to your CartoDB account
+----------------------------------
+
+Perform inserts or updates on your data is simple now using your API
+key. All you need to do, is supply a correct SQL INSERT or UPDATE
+statement for your table along with the api_key parameter for your
+account. Be sure to keep these requests private, as anyone with your API
+key will be able to modify your tables. A correct SQL insert statement
+means that all the columns you want to insert into already exist in
+your table, and all the values for those columns are the right type
+(quoted string, unquoted string for geoms and dates, or numbers).
+
+INSERT
+
+    http://entrypoint?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={Your API key}
+
+Updates are just as simple. Here is an example, updating a row based on
+the value of the cartodb_id column.
+
+UPDATE
+
+    http://entrypoint?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={Your API key}
+
+

npm-shrinkwrap.json

@@ -0,0 +1,247 @@
+{
+  "name": "cartodb_api",
+  "version": "1.1.0",
+  "dependencies": {
+    "cluster2": {
+      "version": "0.3.5-cdb01",
+      "from": "git://github.com/CartoDB/cluster2.git#cdb_production",
+      "dependencies": {
+        "ejs": {
+          "version": "0.8.3"
+        },
+        "npm": {
+          "version": "1.1.62",
+          "dependencies": {
+            "semver": {
+              "version": "1.0.14"
+            },
+            "ini": {
+              "version": "1.0.4"
+            },
+            "slide": {
+              "version": "1.1.3"
+            },
+            "abbrev": {
+              "version": "1.0.3"
+            },
+            "graceful-fs": {
+              "version": "1.1.14"
+            },
+            "minimatch": {
+              "version": "0.2.6"
+            },
+            "nopt": {
+              "version": "2.0.0"
+            },
+            "rimraf": {
+              "version": "2.0.2"
+            },
+            "request": {
+              "version": "2.9.203",
+              "from": "git://github.com/isaacs/request"
+            },
+            "which": {
+              "version": "1.0.5"
+            },
+            "tar": {
+              "version": "0.1.13"
+            },
+            "fstream": {
+              "version": "0.1.19"
+            },
+            "block-stream": {
+              "version": "0.0.6"
+            },
+            "inherits": {
+              "version": "1.0.0",
+              "from": "git://github.com/isaacs/inherits"
+            },
+            "mkdirp": {
+              "version": "0.3.4"
+            },
+            "read": {
+              "version": "1.0.4",
+              "dependencies": {
+                "mute-stream": {
+                  "version": "0.0.3"
+                }
+              }
+            },
+            "lru-cache": {
+              "version": "2.0.4"
+            },
+            "node-gyp": {
+              "version": "0.6.11"
+            },
+            "fstream-npm": {
+              "version": "0.1.2",
+              "dependencies": {
+                "fstream-ignore": {
+                  "version": "0.0.5"
+                }
+              }
+            },
+            "uid-number": {
+              "version": "0.0.3"
+            },
+            "archy": {
+              "version": "0.0.2"
+            },
+            "chownr": {
+              "version": "0.0.1"
+            },
+            "npmlog": {
+              "version": "0.0.2"
+            },
+            "ansi": {
+              "version": "0.1.2"
+            },
+            "npm-registry-client": {
+              "version": "0.2.7"
+            },
+            "read-package-json": {
+              "version": "0.1.5"
+            },
+            "read-installed": {
+              "version": "0.0.2"
+            },
+            "glob": {
+              "version": "3.1.12"
+            },
+            "init-package-json": {
+              "version": "0.0.5",
+              "dependencies": {
+                "promzard": {
+                  "version": "0.2.0"
+                }
+              }
+            },
+            "osenv": {
+              "version": "0.0.3"
+            },
+            "lockfile": {
+              "version": "0.2.1"
+            },
+            "retry": {
+              "version": "0.6.0"
+            },
+            "couch-login": {
+              "version": "0.1.12"
+            },
+            "once": {
+              "version": "1.1.1"
+            },
+            "npmconf": {
+              "version": "0.0.16",
+              "dependencies": {
+                "config-chain": {
+                  "version": "1.1.2",
+                  "dependencies": {
+                    "proto-list": {
+                      "version": "1.2.2"
+                    }
+                  }
+                }
+              }
+            },
+            "opener": {
+              "version": "1.3.0"
+            }
+          }
+        }
+      }
+    },
+    "express": {
+      "version": "2.5.11",
+      "dependencies": {
+        "connect": {
+          "version": "1.9.2",
+          "dependencies": {
+            "formidable": {
+              "version": "1.0.11"
+            }
+          }
+        },
+        "mime": {
+          "version": "1.2.4"
+        },
+        "qs": {
+          "version": "0.4.2"
+        },
+        "mkdirp": {
+          "version": "0.3.0"
+        }
+      }
+    },
+    "underscore": {
+      "version": "1.1.7"
+    },
+    "underscore.string": {
+      "version": "1.1.5",
+      "dependencies": {
+        "underscore": {
+          "version": "1.1.6"
+        }
+      }
+    },
+    "pg": {
+      "version": "0.6.14",
+      "dependencies": {
+        "generic-pool": {
+          "version": "1.0.9"
+        }
+      }
+    },
+    "generic-pool": {
+      "version": "1.0.12"
+    },
+    "redis": {
+      "version": "0.7.1"
+    },
+    "hiredis": {
+      "version": "0.1.14"
+    },
+    "step": {
+      "version": "0.0.5"
+    },
+    "oauth-client": {
+      "version": "0.2.0",
+      "dependencies": {
+        "node-uuid": {
+          "version": "1.1.0"
+        }
+      }
+    },
+    "node-uuid": {
+      "version": "1.3.3"
+    },
+    "csv": {
+      "version": "0.0.13"
+    },
+    "mocha": {
+      "version": "1.2.1",
+      "dependencies": {
+        "commander": {
+          "version": "0.6.1"
+        },
+        "growl": {
+          "version": "1.5.1"
+        },
+        "jade": {
+          "version": "0.26.3",
+          "dependencies": {
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "diff": {
+          "version": "1.0.2"
+        },
+        "debug": {
+          "version": "0.7.0"
+        }
+      }
+    }
+  }
+}

package.json

@@ -2,14 +2,14 @@
     "private": true,
     "name": "cartodb_api",
     "description": "high speed SQL api for cartodb",
-    "version": "1.0.0",
+    "version": "1.1.0",
     "author": {
-        "name": "Simon Tokumine, Vizzuality",
+        "name": "Simon Tokumine, Sandro Santilli, Vizzuality",
         "url": "http://vizzuality.com",
-        "email": "simon@vizzuality.com"
+        "email": "simon@vizzuality.com, strk@vizzuality.com"
     },
     "dependencies": {
-        "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11",
+        "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
         "express": "~2.5.11",
         "underscore"  : "1.1.x",
         "underscore.string": "1.1.5",

test/acceptance/app.test.js

@@ -274,6 +274,82 @@ test('GET /api/v1/sql with SQL parameter and geojson format, ensuring content-di
     });
 });
 
+test('GET /api/v1/sql with SVG format', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
+      format: "svg"
+    });
+    assert.response(app, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.header('Content-Disposition');
+        assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+        assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
+        // TODO: test viewBox
+        done();
+    });
+});
+
+test('GET /api/v1/sql with SVG format and centered point', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ",
+      format: "svg"
+    });
+    assert.response(app, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.header('Content-Disposition');
+        assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+        assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('cx="0" cy="0"') > 0, res.body );
+        // TODO: test viewBox
+        // TODO: test radius
+        done();
+    });
+});
+
+test('GET /api/v1/sql with SVG format and trimmed decimals', function(done){
+    var queryobj = {
+      q: "SELECT 1 as cartodb_id, 'LINESTRING(0 0, 1024 768, 500.123456 600.98765432)'::geometry AS the_geom ",
+      format: "svg",
+      dp: 2
+    };
+    assert.response(app, {
+        url: '/api/v1/sql?' + querystring.stringify(queryobj),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.header('Content-Disposition');
+        assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+        assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.12 167.01" />') > 0, res.body );
+        // TODO: test viewBox
+
+        queryobj.dp = 3;
+        assert.response(app, {
+          url: '/api/v1/sql?' + querystring.stringify(queryobj),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 200, res.body);
+          var cd = res.header('Content-Disposition');
+          assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+          assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
+          assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.123 167.012" />') > 0, res.body );
+          // TODO: test viewBox
+          done();
+        });
+    });
+});
+
 test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){
     assert.response(app, {
         url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
@@ -393,4 +469,18 @@ test('GET decent error if SQL is broken', function(done){
     });
 });
 
+// CSV tests
+test('CSV format', function(done){
+    assert.response(app, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.header('Content-Disposition');
+        assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
+        done();
+    });
+});
+
 });

test/prepare_db.sh

@@ -31,7 +31,7 @@ die() {
 }
 
 echo "preparing postgres..."
-dropdb ${TEST_DB} 2> /dev/null # error expected if doesn't exist
+dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise
 createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database"
 psql -f test.sql ${TEST_DB} 
 

test/run_tests.sh

@@ -28,7 +28,7 @@ echo "port ${REDIS_PORT}" | redis-server - > test/test.log &
 PID_REDIS=$!
 
 echo "Preparing the environment"
-cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -;
+cd test; sh prepare_db.sh || die "database preparation failure"; cd -;
 
 PATH=node_modules/.bin/:$PATH
 

tools/cdbsql

@@ -3,13 +3,25 @@
 // Command line tool for CartoDB SQL API
 //
 // https://github.com/Vizzuality/CartoDB-SQL-API
+//
 
-var http = require('http')
+var http = require('http');
+
+var nodevers = process.versions.node.split('.');
+
+// NOTE: readline is also available in 0.4 but doesn't work
+var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8;
+//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support');
+
+var readline = hasReadline ? require('readline') : null;
 
 var me = process.argv[1];
 
 function usage(exit_code) {
   console.log("Usage: " + me + " [OPTIONS] <query>");
+  if ( hasReadline ) {
+    console.log("       " + me + " [OPTIONS]");
+  }
   console.log("Options:");
   console.log(" -v                      verbose operations (off)");
   console.log(" --help                  print this help");
@@ -18,18 +30,26 @@ function usage(exit_code) {
   console.log(" --port <num>            service tcp port number (8080)");
   console.log(" --api-version <num>     API version (1)");
   console.log(" --key <string>          API authentication key (none)");
+  console.log(" --format <string>       Response format (json)");
+  console.log(" --dp <num>              Decimal places in geojson format (unspecified)");
+  if ( hasReadline ) {
+  console.log(" --batch                 Send all read queries at once (off)");
+  }
   process.exit(exit_code);
 }
 
 process.argv.shift(); // this will be "node" (argv[0])
 process.argv.shift(); // this will be "benchmark.js" (argv[1])
 
+var batch_mode = false;
+var format = 'json';
 var username;
 var domain = 'localhost';
 var port = 8080;
 var api_version = 1;
 var api_key;
 var sql;
+var decimal_places;
 
 var arg;
 while ( arg = process.argv.shift() ) {
@@ -54,6 +74,15 @@ while ( arg = process.argv.shift() ) {
   else if ( arg == '--api-version' ) {
     api_version = process.argv.shift();
   }
+  else if ( arg == '--format' ) {
+    format = process.argv.shift();
+  }
+  else if ( arg == '--dp' ) {
+    decimal_places = process.argv.shift();
+  }
+  else if ( arg == '--batch' ) {
+    batch_mode = true;
+  }
   else if ( ! sql ) {
     sql = arg;
   }
@@ -62,36 +91,106 @@ while ( arg = process.argv.shift() ) {
   }
 }
 
-if ( ! sql ) usage(1);
-
 var hostname = username + '.' + domain;
 
+if ( ! sql ) {
+  if ( readline ) {
+
+    var rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout
+    });
+
+    if ( ! batch_mode ) {
+      rl.setPrompt(hostname + '> ');
+      rl.prompt();
+    }
+
+    sql = '';
+    rl.on('line', function(line) {
+      sql += line;
+      if ( ! batch_mode ) {
+        // TODO: some sanity checking, like trim the line or check if it ends with semicolon
+        if ( sql.length ) {
+          processQuery(sql, function() {
+            sql = '';
+            rl.prompt();
+          });
+        } else rl.prompt();
+      } 
+    }).on('close', function() {
+      if ( batch_mode ) {
+        if ( sql.length )  {
+          processQuery(sql);
+          sql = '';
+        }
+      } else {
+        if ( sql.length ) {
+          console.warn("Unprocessed sql left: [" + sql + "]");
+        }
+        console.log("Good bye");
+      }
+    }).on('SIGCONT', function() {
+      // this is needed so not to exit on stop/resume
+      rl.prompt();
+    });
+  } else {
+    usage(1);
+  }
+} else {
+  processQuery(sql);
+}
+
 // -- Perform the request
 
-var opt = {
+function processQuery(sql, callback)
-  host: hostname,
+{
-  port: port,
-  path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql)
-};
 
-console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path);
+  var post_data = 'q=' + encodeURIComponent(sql);
 
-var body = '';
+  var opt = {
+    host: hostname,
+    port: port,
+    path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format),
+    method: 'POST',
+    headers: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+          'Content-Length': post_data.length
+    }
+  };
 
-http.get(opt, function(res) {
+  if ( typeof(api_key) != 'undefined' ) opt.path += '&api_key=' + api_key;
-  console.log("Response status: " + res.statusCode);
+  if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places;
-  res.on('data', function(chunk) {
+
-    body += chunk;
+  var body = '';
-    //console.log("data: "); console.dir(json);
+  var request = 'http://' + opt.host + ':' + opt.port + opt.path;
+  //console.log("Sending request:", request);
+
+  var req = http.request(opt, function(res) {
+    //console.log("Response status: " + res.statusCode);
+    res.on('data', function(chunk) {
+      body += chunk;
+      //console.log("data: "); console.dir(json);
+    });
+
+    res.on('end', function() {
+      console.log("Request:", request);
+      var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql;
+      sqlprint = sqlprint.split('\n').join(' ');
+      console.log("Query:", sqlprint);
+      console.log("Response status: " + res.statusCode);
+      console.log('Response body:');
+      console.dir(body);
+      if ( callback ) callback();
+    });
+
+  }).on('error', function(e) {
+    console.log("Request:", request);
+    console.log("Error: " + e.message);
+    if ( callback ) callback();
   });
 
-  res.on('end', function() {
+  req.write(post_data);
-    console.log('Body:');
+  req.end();
-    var json = JSON.parse(body);
-    console.dir(json);
-  });
-}).on('error', function(e) {
-  console.log("ERROR: " + e.message);
-});
-
 
+}