cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/staging'

Browse Source 
Conflicts:
	NEWS.md
	package.json
This commit is contained in:
Luis Bosque 2012-10-30 13:13:56 +01:00
commit fae656fe09
18 changed files with 808 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
config/environments/*.js
logs/*.log logs/*.log
pids/*.pid pids/*.pid
*.sock *.sock

11
NEWS.md
View File

@ -1,3 +1,14 @@
1.1.0 (30/10/12)
-----
* Fixed problem in cluster2 with pidfile name
* SVG output format
* Enhancement to the cdbsql tool:
- New switches: --format, --key, --dp
- Interactive mode
* API documentation
* ./configure script
* Restrict listening to a node host
1.0.0 (03/10/12) 1.0.0 (03/10/12)
----- -----
* Migrated to node 0.8 version * Migrated to node 0.8 version

47
README.md
View File

@ -10,26 +10,11 @@ Provides a nodejs based API for running SQL queries against CartoDB.
core requirements core requirements
------------- -------------
* postgres 9.0+ * postgres 9.0+
* cartodb 0.9.5+ (for CDB_QueryTables) * cartodb 0.9.5+ (for ``CDB_QueryTables``)
* redis * redis
* node > v0.4.8 && < v0.9.0 * node 0.8+
* npm * npm
usage
-----
Edit config/environments/<environment>.js
Make sure redis is running and knows about active cartodb user.
``` bash
node [cluster.js|app.js] <environment>
```
Supported <environment> values are developement, test, production
for examples of use, see /tests
Install dependencies Install dependencies
--------------------- ---------------------
@ -37,12 +22,32 @@ Install dependencies
npm install npm install
``` ```
usage
-----
Create and edit config/environments/<environment>.js from .js.example files.
You may find the ./configure script useful to make an edited copy for you,
see ```./configure --help``` for a list of supported switches.
Make sure redis is running and knows about active cartodb user.
Make sure your PostgreSQL server is running, is accessible on
the host and port specified in the <environment> file, has
a 'publicuser' role and trusts user authentication from localhost
connections.
``` bash
node [cluster.js|app.js] <environment>
```
Supported <environment> values are developement, test, production
See doc/API.md for API documentation.
For examples of use, see under test/.
tests tests
------ ------
see test/README.md
Run ```make check``` or see test/README.md
note on 0.4.x
--------------
output of large result sets is slow under node 0.4. Recommend running under 0.6 where possible.

5
app.js
View File

@ -26,5 +26,6 @@ _.extend(global.settings, env);
// kick off controller // kick off controller
var app = require(global.settings.app_root + '/app/controllers/app'); var app = require(global.settings.app_root + '/app/controllers/app');
app.listen(global.settings.node_port); app.listen(global.settings.node_port, global.settings.node_host, function() {
console.log("CartoDB SQL API listening on port " + global.settings.node_port); console.log("CartoDB SQL API listening on " + global.settings.node_host + ":" + global.settings.node_port);
});

132
app/controllers/app.js
View File

@ -12,6 +12,8 @@
// - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public // - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public
// //
// eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table // eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table
//
//
var express = require('express') var express = require('express')
, app = express.createServer( , app = express.createServer(
express.logger({ express.logger({
@ -47,7 +49,11 @@ function handleQuery(req, res) {
var limit = parseInt(req.query.rows_per_page); var limit = parseInt(req.query.rows_per_page);
var offset = parseInt(req.query.page); var offset = parseInt(req.query.page);
var format = req.query.format; var format = req.query.format;
var dp = req.query.dp; var dp = req.query.dp; // decimal point digits (defaults to 6)
var gn = "the_geom"; // TODO: read from configuration file
var svg_width = 1024.0;
var svg_height = 768.0;
// sanitize and apply defaults to input // sanitize and apply defaults to input
dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp; dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp;
@ -127,6 +133,27 @@ function handleQuery(req, res) {
// TODO: refactor formats to external object // TODO: refactor formats to external object
if (format === 'geojson'){ if (format === 'geojson'){
sql = ['SELECT *, ST_AsGeoJSON(the_geom,',dp,') as the_geom FROM (', sql, ') as foo'].join(""); sql = ['SELECT *, ST_AsGeoJSON(the_geom,',dp,') as the_geom FROM (', sql, ') as foo'].join("");
} else if (format === 'svg'){
var svg_ratio = svg_width/svg_height;
sql = 'WITH source AS ( ' + sql + '), extent AS ( '
+ ' SELECT ST_Extent(' + gn + ') AS e FROM source '
+ '), extent_info AS ( SELECT e, '
+ 'st_xmin(e) as ex0, st_ymax(e) as ey0, '
+ 'st_xmax(e)-st_xmin(e) as ew, '
+ 'st_ymax(e)-st_ymin(e) as eh FROM extent )'
+ ', trans AS ( SELECT CASE WHEN '
+ 'eh = 0 THEN ' + svg_width
+ '/ COALESCE(NULLIF(ew,0),' + svg_width +') WHEN '
+ svg_ratio + ' <= (ew / eh) THEN ('
+ svg_width + '/ew ) ELSE ('
+ svg_height + '/eh ) END as s '
+ ', ex0 as x0, ey0 as y0 FROM extent_info ) '
+ 'SELECT st_TransScale(e, -x0, -y0, s, s)::box2d as '
+ gn + '_box, ST_Dimension(' + gn + ') as ' + gn
+ '_dimension, ST_AsSVG(ST_TransScale(' + gn + ', '
+ '-x0, -y0, s, s), 0, ' + dp + ') as ' + gn
//+ ', ex0, ey0, ew, eh, s ' // DEBUG ONLY
+ ' FROM trans, extent_info, source';
} }
pg.query(sql, this); pg.query(sql, this);
@ -154,9 +181,12 @@ function handleQuery(req, res) {
// TODO: refactor formats to external object // TODO: refactor formats to external object
if (format === 'geojson'){ if (format === 'geojson'){
toGeoJSON(result, res, this); toGeoJSON(result, res, this);
} else if (format === 'svg'){
toSVG(result.rows, gn, this);
} else if (format === 'csv'){ } else if (format === 'csv'){
toCSV(result, res, this); toCSV(result, res, this);
} else { } else {
// TODO: error out if 'format' resolves to an unsupported format !
var end = new Date().getTime(); var end = new Date().getTime();
var json_result = {'time' : (end - start)/1000}; var json_result = {'time' : (end - start)/1000};
@ -218,6 +248,98 @@ function toGeoJSON(data, res, callback){
} }
} }
function toSVG(rows, gn, callback){
var radius = 5; // in pixels (based on svg_width and svg_height)
var stroke_width = 1; // in pixels (based on svg_width and svg_height)
var stroke_color = 'black';
// fill settings affect polygons and points (circles)
var fill_opacity = 0.5; // 0.0 is fully transparent, 1.0 is fully opaque
// unused if fill_color='none'
var fill_color = 'none'; // affects polygons and circles
var bbox; // will be computed during the results scan
var polys = [];
var lines = [];
var points = [];
_.each(rows, function(ele){
var g = ele[gn];
if ( ! g ) return; // null or empty
var gdims = ele[gn + '_dimension'];
// TODO: add an identifier, if any of "cartodb_id", "oid", "id", "gid" are found
// TODO: add "class" attribute to help with styling ?
if ( gdims == '0' ) {
points.push('<circle r="[RADIUS]" ' + g + ' />');
} else if ( gdims == '1' ) {
// Avoid filling closed linestrings
var linetag = '<path ';
if ( fill_color != 'none' ) linetag += 'fill="none" '
linetag += 'd="' + g + '" />';
lines.push(linetag);
} else if ( gdims == '2' ) {
polys.push('<path d="' + g + '" />');
}
if ( ! bbox ) {
// Parse layer extent: "BOX(x y, X Y)"
// NOTE: the name of the extent field is
// determined by the same code adding the
// ST_AsSVG call (in queryResult)
//
bbox = ele[gn + '_box'];
bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/);
bbox = {
xmin: parseFloat(bbox[1]),
ymin: parseFloat(bbox[2]),
xmax: parseFloat(bbox[3]),
ymax: parseFloat(bbox[4])
};
}
});
// Set point radius
for (var i=0; i<points.length; ++i) {
points[i] = points[i].replace('[RADIUS]', radius);
}
var header_tags = [
'<?xml version="1.0" standalone="no"?>',
'<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">',
];
var root_tag = '<svg ';
if ( bbox ) {
// expand box by "radius" + "stroke-width"
// TODO: use a Box2d class for these ops
var growby = radius+stroke_width;
bbox.xmin -= growby;
bbox.ymin -= growby;
bbox.xmax += growby;
bbox.ymax += growby;
bbox.width = bbox.xmax - bbox.xmin;
bbox.height = bbox.ymax - bbox.ymin;
root_tag += 'viewBox="' + bbox.xmin + ' ' + (-bbox.ymax) + ' '
+ bbox.width + ' ' + bbox.height + '" ';
}
root_tag += 'style="fill-opacity:' + fill_opacity
+ '; stroke:' + stroke_color
+ '; stroke-width:' + stroke_width
+ '; fill:' + fill_color
+ '" ';
root_tag += 'xmlns="http://www.w3.org/2000/svg" version="1.1">';
header_tags.push(root_tag);
// Render points on top of lines and lines on top of polys
var out = header_tags.concat(polys, lines, points);
out.push('</svg>');
// return payload
callback(null, out.join("\n"));
}
function toCSV(data, res, callback){ function toCSV(data, res, callback){
try{ try{
// pull out keys for column headers // pull out keys for column headers
@ -238,9 +360,12 @@ function getContentDisposition(format){
if (format === 'geojson'){ if (format === 'geojson'){
ext = 'geojson'; ext = 'geojson';
} }
if (format === 'csv'){ else if (format === 'csv'){
ext = 'csv'; ext = 'csv';
} }
else if (format === 'svg'){
ext = 'svg';
}
var time = new Date().toUTCString(); var time = new Date().toUTCString();
return 'inline; filename=cartodb-query.' + ext + '; modification-date="' + time + '";'; return 'inline; filename=cartodb-query.' + ext + '; modification-date="' + time + '";';
} }
@ -250,6 +375,9 @@ function getContentType(format){
if (format === 'csv'){ if (format === 'csv'){
type = "text/csv; charset=utf-8"; type = "text/csv; charset=utf-8";
} }
else if (format === 'svg'){
type = "image/svg+xml; charset=utf-8";
}
return type; return type;
} }

6
cluster.js
View File

@ -30,11 +30,13 @@ var app = require(global.settings.app_root + '/app/controllers/app');
var cluster = new Cluster({ var cluster = new Cluster({
port: global.settings.node_port, port: global.settings.node_port,
host: global.settings.node_host,
monHost: global.settings.node_host,
monPort: global.settings.node_port+1 monPort: global.settings.node_port+1
}); });
cluster.listen(function(cb) { cluster.listen(function(cb) {
cb(app); cb(app);
}, function() {
console.log("CartoDB SQL API listening on " + global.settings.node_host + ':' + global.settings.node_port);
}); });
console.log("CartoDB SQL API listening on port " + global.settings.node_port);

1
config/environments/development.js → config/environments/development.js.example
View File

@ -1,4 +1,5 @@
module.exports.node_port = 8080; module.exports.node_port = 8080;
module.exports.node_host = '127.0.0.1';
module.exports.environment = 'development'; module.exports.environment = 'development';
module.exports.db_base_name = 'cartodb_dev_user_<%= user_id %>_db'; module.exports.db_base_name = 'cartodb_dev_user_<%= user_id %>_db';
module.exports.db_user = 'development_cartodb_user_<%= user_id %>'; module.exports.db_user = 'development_cartodb_user_<%= user_id %>';

0
config/environments/production.js → config/environments/production.js.example
View File

0
config/environments/staging.js → config/environments/staging.js.example
View File

0
config/environments/test.js → config/environments/test.js.example
View File

37
configure vendored Executable file
View File

@ -0,0 +1,37 @@
#!/bin/sh
usage() {
echo "Usage: $0 [OPTION]"
echo
echo "Configuration:"
echo " --help display this help and exit"
echo " --with-pgport=NUM access PostgreSQL server on TCP port NUM"
}
PGPORT=5432
while test -n "$1"; do
case "$1" in
--help|-h)
usage
exit 0
;;
--with-pgport=*)
PGPORT=`echo "$1" | cut -d= -f2`
;;
*)
echo "Unknown option '$1'" >&2
usage >&2
exit 1
esac
shift
done
echo "PGPORT: $PGPORT"
# TODO: allow specifying configuration settings !
for f in config/environments/*.example; do
o=`dirname "$f"`/`basename "$f" .example`
echo "Writing $o"
sed "s/\( *module.exports.db_port[ \t]*= *'\?\)[^';]*\('\?;\)/\1$PGPORT\2/" < "$f" > "$o"
done

127
doc/API.md Normal file
View File

@ -0,0 +1,127 @@
SQL API
=======
Request format
--------------
Supported query string parameters:
'q': Specifies the SQL query to run
Example:
'http://entrypoint?q=SELECT count(*) FROM mytable'
'format': Specifies which format to use for the response.
Supported formats: JSON (the default), GeoJSON,
CSV, SVG
'dp': Number of digits after the decimal point.
Only affects format=GeoJSON.
By default this is 6.
'api_key': Needed to authenticate in order to modify the database.
Response formats
----------------
The standard response from the CartoDB SQL API is JSON. If you are
building a web-application, the lightweight JSON format allows you to
quickly integrate data from the SQL API.
The JSON response is as follows:
```
{
time: 0.006,
total_rows: 1,
rows: [
{
year: " 2011",
the_geom: "0101000020E610...",
cartodb_id: 1,
created_at: "2012-02-06T22:50:35.778Z",
updated_at: "2012-02-12T21:34:08.193Z"
}
]
}
```
Alternatively, you can use the GeoJSON specification for returning data
from the API. To do so, simply supply the format parameter as GeoJSON.
The GeoJSON response is follows:
```
{
type: "FeatureCollection",
features: [
{
type: "Feature",
properties: {
year: " 2011",
month: 10,
day: "11",
cartodb_id: 1,
created_at: "2012-02-06T22:50:35.778Z",
updated_at: "2012-02-12T21:34:08.193Z"
},
geometry: {
type: "Point",
coordinates: [
-97.335,
35.498
]
}
}
]
}
```
TODO: csv, kml responses
Response errors
---------------
To help you debug your SQL queries, the CartoDB SQL API returns errors
as part of the JSON response. Errors come back as follows,
```
{
error: [
"syntax error at or near "LIMIT""
]
}
```
You can use these errors to help understand your SQL.
Getting table information
-------------------------
Currently, there is no public method for accessing your table schemas. The
simplest way to get table structure is to access the first row of the data:
http://entrypoint?q=SELECT * FROM mytable LIMIT 1
Write data to your CartoDB account
----------------------------------
Perform inserts or updates on your data is simple now using your API
key. All you need to do, is supply a correct SQL INSERT or UPDATE
statement for your table along with the api_key parameter for your
account. Be sure to keep these requests private, as anyone with your API
key will be able to modify your tables. A correct SQL insert statement
means that all the columns you want to insert into already exist in
your table, and all the values for those columns are the right type
(quoted string, unquoted string for geoms and dates, or numbers).
INSERT
http://entrypoint?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={Your API key}
Updates are just as simple. Here is an example, updating a row based on
the value of the cartodb_id column.
UPDATE
http://entrypoint?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={Your API key}

247
npm-shrinkwrap.json generated Normal file
View File

@ -0,0 +1,247 @@
{
"name": "cartodb_api",
"version": "1.1.0",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb01",
"from": "git://github.com/CartoDB/cluster2.git#cdb_production",
"dependencies": {
"ejs": {
"version": "0.8.3"
},
"npm": {
"version": "1.1.62",
"dependencies": {
"semver": {
"version": "1.0.14"
},
"ini": {
"version": "1.0.4"
},
"slide": {
"version": "1.1.3"
},
"abbrev": {
"version": "1.0.3"
},
"graceful-fs": {
"version": "1.1.14"
},
"minimatch": {
"version": "0.2.6"
},
"nopt": {
"version": "2.0.0"
},
"rimraf": {
"version": "2.0.2"
},
"request": {
"version": "2.9.203",
"from": "git://github.com/isaacs/request"
},
"which": {
"version": "1.0.5"
},
"tar": {
"version": "0.1.13"
},
"fstream": {
"version": "0.1.19"
},
"block-stream": {
"version": "0.0.6"
},
"inherits": {
"version": "1.0.0",
"from": "git://github.com/isaacs/inherits"
},
"mkdirp": {
"version": "0.3.4"
},
"read": {
"version": "1.0.4",
"dependencies": {
"mute-stream": {
"version": "0.0.3"
}
}
},
"lru-cache": {
"version": "2.0.4"
},
"node-gyp": {
"version": "0.6.11"
},
"fstream-npm": {
"version": "0.1.2",
"dependencies": {
"fstream-ignore": {
"version": "0.0.5"
}
}
},
"uid-number": {
"version": "0.0.3"
},
"archy": {
"version": "0.0.2"
},
"chownr": {
"version": "0.0.1"
},
"npmlog": {
"version": "0.0.2"
},
"ansi": {
"version": "0.1.2"
},
"npm-registry-client": {
"version": "0.2.7"
},
"read-package-json": {
"version": "0.1.5"
},
"read-installed": {
"version": "0.0.2"
},
"glob": {
"version": "3.1.12"
},
"init-package-json": {
"version": "0.0.5",
"dependencies": {
"promzard": {
"version": "0.2.0"
}
}
},
"osenv": {
"version": "0.0.3"
},
"lockfile": {
"version": "0.2.1"
},
"retry": {
"version": "0.6.0"
},
"couch-login": {
"version": "0.1.12"
},
"once": {
"version": "1.1.1"
},
"npmconf": {
"version": "0.0.16",
"dependencies": {
"config-chain": {
"version": "1.1.2",
"dependencies": {
"proto-list": {
"version": "1.2.2"
}
}
}
}
},
"opener": {
"version": "1.3.0"
}
}
}
}
},
"express": {
"version": "2.5.11",
"dependencies": {
"connect": {
"version": "1.9.2",
"dependencies": {
"formidable": {
"version": "1.0.11"
}
}
},
"mime": {
"version": "1.2.4"
},
"qs": {
"version": "0.4.2"
},
"mkdirp": {
"version": "0.3.0"
}
}
},
"underscore": {
"version": "1.1.7"
},
"underscore.string": {
"version": "1.1.5",
"dependencies": {
"underscore": {
"version": "1.1.6"
}
}
},
"pg": {
"version": "0.6.14",
"dependencies": {
"generic-pool": {
"version": "1.0.9"
}
}
},
"generic-pool": {
"version": "1.0.12"
},
"redis": {
"version": "0.7.1"
},
"hiredis": {
"version": "0.1.14"
},
"step": {
"version": "0.0.5"
},
"oauth-client": {
"version": "0.2.0",
"dependencies": {
"node-uuid": {
"version": "1.1.0"
}
}
},
"node-uuid": {
"version": "1.3.3"
},
"csv": {
"version": "0.0.13"
},
"mocha": {
"version": "1.2.1",
"dependencies": {
"commander": {
"version": "0.6.1"
},
"growl": {
"version": "1.5.1"
},
"jade": {
"version": "0.26.3",
"dependencies": {
"mkdirp": {
"version": "0.3.0"
}
}
},
"diff": {
"version": "1.0.2"
},
"debug": {
"version": "0.7.0"
}
}
}
}
}

8
package.json
View File

@ -2,14 +2,14 @@
"private": true, "private": true,
"name": "cartodb_api", "name": "cartodb_api",
"description": "high speed SQL api for cartodb", "description": "high speed SQL api for cartodb",
"version": "1.0.0", "version": "1.1.0",
"author": { "author": {
"name": "Simon Tokumine, Vizzuality", "name": "Simon Tokumine, Sandro Santilli, Vizzuality",
"url": "http://vizzuality.com", "url": "http://vizzuality.com",
"email": "simon@vizzuality.com" "email": "simon@vizzuality.com, strk@vizzuality.com"
}, },
"dependencies": { "dependencies": {
"cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
"express": "~2.5.11", "express": "~2.5.11",
"underscore" : "1.1.x", "underscore" : "1.1.x",
"underscore.string": "1.1.5", "underscore.string": "1.1.5",

90
test/acceptance/app.test.js
View File

@ -274,6 +274,82 @@ test('GET /api/v1/sql with SQL parameter and geojson format, ensuring content-di
}); });
}); });
test('GET /api/v1/sql with SVG format', function(done){
var query = querystring.stringify({
q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
format: "svg"
});
assert.response(app, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
// TODO: test viewBox
done();
});
});
test('GET /api/v1/sql with SVG format and centered point', function(done){
var query = querystring.stringify({
q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ",
format: "svg"
});
assert.response(app, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('cx="0" cy="0"') > 0, res.body );
// TODO: test viewBox
// TODO: test radius
done();
});
});
test('GET /api/v1/sql with SVG format and trimmed decimals', function(done){
var queryobj = {
q: "SELECT 1 as cartodb_id, 'LINESTRING(0 0, 1024 768, 500.123456 600.98765432)'::geometry AS the_geom ",
format: "svg",
dp: 2
};
assert.response(app, {
url: '/api/v1/sql?' + querystring.stringify(queryobj),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.12 167.01" />') > 0, res.body );
// TODO: test viewBox
queryobj.dp = 3;
assert.response(app, {
url: '/api/v1/sql?' + querystring.stringify(queryobj),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.123 167.012" />') > 0, res.body );
// TODO: test viewBox
done();
});
});
});
test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){ test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){
assert.response(app, { assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4', url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
@ -393,4 +469,18 @@ test('GET decent error if SQL is broken', function(done){
}); });
}); });
// CSV tests
test('CSV format', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
done();
});
});
}); });

2
test/prepare_db.sh
View File

@ -31,7 +31,7 @@ die() {
} }
echo "preparing postgres..." echo "preparing postgres..."
dropdb ${TEST_DB} 2> /dev/null # error expected if doesn't exist dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise
createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database" createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database"
psql -f test.sql ${TEST_DB} psql -f test.sql ${TEST_DB}

2
test/run_tests.sh
View File

@ -28,7 +28,7 @@ echo "port ${REDIS_PORT}" | redis-server - > test/test.log &
PID_REDIS=$! PID_REDIS=$!
echo "Preparing the environment" echo "Preparing the environment"
cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -; cd test; sh prepare_db.sh || die "database preparation failure"; cd -;
PATH=node_modules/.bin/:$PATH PATH=node_modules/.bin/:$PATH

147
tools/cdbsql
View File

@ -3,13 +3,25 @@
// Command line tool for CartoDB SQL API // Command line tool for CartoDB SQL API
// //
// https://github.com/Vizzuality/CartoDB-SQL-API // https://github.com/Vizzuality/CartoDB-SQL-API
//
var http = require('http') var http = require('http');
var nodevers = process.versions.node.split('.');
// NOTE: readline is also available in 0.4 but doesn't work
var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8;
//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support');
var readline = hasReadline ? require('readline') : null;
var me = process.argv[1]; var me = process.argv[1];
function usage(exit_code) { function usage(exit_code) {
console.log("Usage: " + me + " [OPTIONS] <query>"); console.log("Usage: " + me + " [OPTIONS] <query>");
if ( hasReadline ) {
console.log(" " + me + " [OPTIONS]");
}
console.log("Options:"); console.log("Options:");
console.log(" -v verbose operations (off)"); console.log(" -v verbose operations (off)");
console.log(" --help print this help"); console.log(" --help print this help");
@ -18,18 +30,26 @@ function usage(exit_code) {
console.log(" --port <num> service tcp port number (8080)"); console.log(" --port <num> service tcp port number (8080)");
console.log(" --api-version <num> API version (1)"); console.log(" --api-version <num> API version (1)");
console.log(" --key <string> API authentication key (none)"); console.log(" --key <string> API authentication key (none)");
console.log(" --format <string> Response format (json)");
console.log(" --dp <num> Decimal places in geojson format (unspecified)");
if ( hasReadline ) {
console.log(" --batch Send all read queries at once (off)");
}
process.exit(exit_code); process.exit(exit_code);
} }
process.argv.shift(); // this will be "node" (argv[0]) process.argv.shift(); // this will be "node" (argv[0])
process.argv.shift(); // this will be "benchmark.js" (argv[1]) process.argv.shift(); // this will be "benchmark.js" (argv[1])
var batch_mode = false;
var format = 'json';
var username; var username;
var domain = 'localhost'; var domain = 'localhost';
var port = 8080; var port = 8080;
var api_version = 1; var api_version = 1;
var api_key; var api_key;
var sql; var sql;
var decimal_places;
var arg; var arg;
while ( arg = process.argv.shift() ) { while ( arg = process.argv.shift() ) {
@ -54,6 +74,15 @@ while ( arg = process.argv.shift() ) {
else if ( arg == '--api-version' ) { else if ( arg == '--api-version' ) {
api_version = process.argv.shift(); api_version = process.argv.shift();
} }
else if ( arg == '--format' ) {
format = process.argv.shift();
}
else if ( arg == '--dp' ) {
decimal_places = process.argv.shift();
}
else if ( arg == '--batch' ) {
batch_mode = true;
}
else if ( ! sql ) { else if ( ! sql ) {
sql = arg; sql = arg;
} }
@ -62,36 +91,106 @@ while ( arg = process.argv.shift() ) {
} }
} }
if ( ! sql ) usage(1);
var hostname = username + '.' + domain; var hostname = username + '.' + domain;
if ( ! sql ) {
if ( readline ) {
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
if ( ! batch_mode ) {
rl.setPrompt(hostname + '> ');
rl.prompt();
}
sql = '';
rl.on('line', function(line) {
sql += line;
if ( ! batch_mode ) {
// TODO: some sanity checking, like trim the line or check if it ends with semicolon
if ( sql.length ) {
processQuery(sql, function() {
sql = '';
rl.prompt();
});
} else rl.prompt();
}
}).on('close', function() {
if ( batch_mode ) {
if ( sql.length ) {
processQuery(sql);
sql = '';
}
} else {
if ( sql.length ) {
console.warn("Unprocessed sql left: [" + sql + "]");
}
console.log("Good bye");
}
}).on('SIGCONT', function() {
// this is needed so not to exit on stop/resume
rl.prompt();
});
} else {
usage(1);
}
} else {
processQuery(sql);
}
// -- Perform the request // -- Perform the request
var opt = { function processQuery(sql, callback)
host: hostname, {
port: port,
path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql)
};
console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path); var post_data = 'q=' + encodeURIComponent(sql);
var body = ''; var opt = {
host: hostname,
port: port,
path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format),
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': post_data.length
}
};
http.get(opt, function(res) { if ( typeof(api_key) != 'undefined' ) opt.path += '&api_key=' + api_key;
console.log("Response status: " + res.statusCode); if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places;
res.on('data', function(chunk) {
body += chunk; var body = '';
//console.log("data: "); console.dir(json); var request = 'http://' + opt.host + ':' + opt.port + opt.path;
//console.log("Sending request:", request);
var req = http.request(opt, function(res) {
//console.log("Response status: " + res.statusCode);
res.on('data', function(chunk) {
body += chunk;
//console.log("data: "); console.dir(json);
});
res.on('end', function() {
console.log("Request:", request);
var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql;
sqlprint = sqlprint.split('\n').join(' ');
console.log("Query:", sqlprint);
console.log("Response status: " + res.statusCode);
console.log('Response body:');
console.dir(body);
if ( callback ) callback();
});
}).on('error', function(e) {
console.log("Request:", request);
console.log("Error: " + e.message);
if ( callback ) callback();
}); });
res.on('end', function() { req.write(post_data);
console.log('Body:'); req.end();
var json = JSON.parse(body);
console.dir(json);
});
}).on('error', function(e) {
console.log("ERROR: " + e.message);
});
}