diff --git a/.gitignore b/.gitignore index ae7c93a0..f38ce45e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ +config/environments/*.js logs/*.log pids/*.pid *.sock test/tmp/* node_modules/ -.idea/* \ No newline at end of file +.idea/* diff --git a/NEWS.md b/NEWS.md index b47bed08..1d3e3e93 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,14 @@ +1.1.0 (30/10/12) +----- +* Fixed problem in cluster2 with pidfile name +* SVG output format +* Enhancement to the cdbsql tool: + - New switches: --format, --key, --dp + - Interactive mode +* API documentation +* ./configure script +* Restrict listening to a node host + 1.0.0 (03/10/12) ----- * Migrated to node 0.8 version diff --git a/README.md b/README.md index 568ed750..cc36d107 100644 --- a/README.md +++ b/README.md @@ -10,26 +10,11 @@ Provides a nodejs based API for running SQL queries against CartoDB. core requirements ------------- * postgres 9.0+ -* cartodb 0.9.5+ (for CDB_QueryTables) +* cartodb 0.9.5+ (for ``CDB_QueryTables``) * redis -* node > v0.4.8 && < v0.9.0 +* node 0.8+ * npm -usage ------ - -Edit config/environments/.js -Make sure redis is running and knows about active cartodb user. - -``` bash -node [cluster.js|app.js] -``` - -Supported values are developement, test, production - -for examples of use, see /tests - - Install dependencies --------------------- @@ -37,12 +22,32 @@ Install dependencies npm install ``` +usage +----- + +Create and edit config/environments/.js from .js.example files. +You may find the ./configure script useful to make an edited copy for you, +see ```./configure --help``` for a list of supported switches. + +Make sure redis is running and knows about active cartodb user. + +Make sure your PostgreSQL server is running, is accessible on +the host and port specified in the file, has +a 'publicuser' role and trusts user authentication from localhost +connections. + +``` bash +node [cluster.js|app.js] +``` + +Supported values are developement, test, production + +See doc/API.md for API documentation. +For examples of use, see under test/. + tests ------ -see test/README.md +Run ```make check``` or see test/README.md -note on 0.4.x --------------- -output of large result sets is slow under node 0.4. Recommend running under 0.6 where possible. diff --git a/app.js b/app.js index 21de716b..58c9cf6b 100755 --- a/app.js +++ b/app.js @@ -26,5 +26,6 @@ _.extend(global.settings, env); // kick off controller var app = require(global.settings.app_root + '/app/controllers/app'); -app.listen(global.settings.node_port); -console.log("CartoDB SQL API listening on port " + global.settings.node_port); \ No newline at end of file +app.listen(global.settings.node_port, global.settings.node_host, function() { + console.log("CartoDB SQL API listening on " + global.settings.node_host + ":" + global.settings.node_port); +}); diff --git a/app/controllers/app.js b/app/controllers/app.js index e87172b0..9e9f33bc 100755 --- a/app/controllers/app.js +++ b/app/controllers/app.js @@ -12,6 +12,8 @@ // - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public // // eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table +// +// var express = require('express') , app = express.createServer( express.logger({ @@ -47,7 +49,11 @@ function handleQuery(req, res) { var limit = parseInt(req.query.rows_per_page); var offset = parseInt(req.query.page); var format = req.query.format; - var dp = req.query.dp; + var dp = req.query.dp; // decimal point digits (defaults to 6) + var gn = "the_geom"; // TODO: read from configuration file + var svg_width = 1024.0; + var svg_height = 768.0; + // sanitize and apply defaults to input dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp; @@ -127,6 +133,27 @@ function handleQuery(req, res) { // TODO: refactor formats to external object if (format === 'geojson'){ sql = ['SELECT *, ST_AsGeoJSON(the_geom,',dp,') as the_geom FROM (', sql, ') as foo'].join(""); + } else if (format === 'svg'){ + var svg_ratio = svg_width/svg_height; + sql = 'WITH source AS ( ' + sql + '), extent AS ( ' + + ' SELECT ST_Extent(' + gn + ') AS e FROM source ' + + '), extent_info AS ( SELECT e, ' + + 'st_xmin(e) as ex0, st_ymax(e) as ey0, ' + + 'st_xmax(e)-st_xmin(e) as ew, ' + + 'st_ymax(e)-st_ymin(e) as eh FROM extent )' + + ', trans AS ( SELECT CASE WHEN ' + + 'eh = 0 THEN ' + svg_width + + '/ COALESCE(NULLIF(ew,0),' + svg_width +') WHEN ' + + svg_ratio + ' <= (ew / eh) THEN (' + + svg_width + '/ew ) ELSE (' + + svg_height + '/eh ) END as s ' + + ', ex0 as x0, ey0 as y0 FROM extent_info ) ' + + 'SELECT st_TransScale(e, -x0, -y0, s, s)::box2d as ' + + gn + '_box, ST_Dimension(' + gn + ') as ' + gn + + '_dimension, ST_AsSVG(ST_TransScale(' + gn + ', ' + + '-x0, -y0, s, s), 0, ' + dp + ') as ' + gn + //+ ', ex0, ey0, ew, eh, s ' // DEBUG ONLY + + ' FROM trans, extent_info, source'; } pg.query(sql, this); @@ -154,9 +181,12 @@ function handleQuery(req, res) { // TODO: refactor formats to external object if (format === 'geojson'){ toGeoJSON(result, res, this); + } else if (format === 'svg'){ + toSVG(result.rows, gn, this); } else if (format === 'csv'){ toCSV(result, res, this); } else { + // TODO: error out if 'format' resolves to an unsupported format ! var end = new Date().getTime(); var json_result = {'time' : (end - start)/1000}; @@ -218,6 +248,98 @@ function toGeoJSON(data, res, callback){ } } +function toSVG(rows, gn, callback){ + + var radius = 5; // in pixels (based on svg_width and svg_height) + var stroke_width = 1; // in pixels (based on svg_width and svg_height) + var stroke_color = 'black'; + // fill settings affect polygons and points (circles) + var fill_opacity = 0.5; // 0.0 is fully transparent, 1.0 is fully opaque + // unused if fill_color='none' + var fill_color = 'none'; // affects polygons and circles + + var bbox; // will be computed during the results scan + var polys = []; + var lines = []; + var points = []; + _.each(rows, function(ele){ + var g = ele[gn]; + if ( ! g ) return; // null or empty + var gdims = ele[gn + '_dimension']; + + // TODO: add an identifier, if any of "cartodb_id", "oid", "id", "gid" are found + // TODO: add "class" attribute to help with styling ? + if ( gdims == '0' ) { + points.push(''); + } else if ( gdims == '1' ) { + // Avoid filling closed linestrings + var linetag = ''; + lines.push(linetag); + } else if ( gdims == '2' ) { + polys.push(''); + } + + if ( ! bbox ) { + // Parse layer extent: "BOX(x y, X Y)" + // NOTE: the name of the extent field is + // determined by the same code adding the + // ST_AsSVG call (in queryResult) + // + bbox = ele[gn + '_box']; + bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/); + bbox = { + xmin: parseFloat(bbox[1]), + ymin: parseFloat(bbox[2]), + xmax: parseFloat(bbox[3]), + ymax: parseFloat(bbox[4]) + }; + } + }); + + // Set point radius + for (var i=0; i', + '', + ]; + + var root_tag = ''); + + // return payload + callback(null, out.join("\n")); +} + function toCSV(data, res, callback){ try{ // pull out keys for column headers @@ -238,9 +360,12 @@ function getContentDisposition(format){ if (format === 'geojson'){ ext = 'geojson'; } - if (format === 'csv'){ + else if (format === 'csv'){ ext = 'csv'; } + else if (format === 'svg'){ + ext = 'svg'; + } var time = new Date().toUTCString(); return 'inline; filename=cartodb-query.' + ext + '; modification-date="' + time + '";'; } @@ -250,6 +375,9 @@ function getContentType(format){ if (format === 'csv'){ type = "text/csv; charset=utf-8"; } + else if (format === 'svg'){ + type = "image/svg+xml; charset=utf-8"; + } return type; } diff --git a/cluster.js b/cluster.js index bc705630..30417979 100755 --- a/cluster.js +++ b/cluster.js @@ -30,11 +30,13 @@ var app = require(global.settings.app_root + '/app/controllers/app'); var cluster = new Cluster({ port: global.settings.node_port, + host: global.settings.node_host, + monHost: global.settings.node_host, monPort: global.settings.node_port+1 }); cluster.listen(function(cb) { cb(app); +}, function() { + console.log("CartoDB SQL API listening on " + global.settings.node_host + ':' + global.settings.node_port); }); - -console.log("CartoDB SQL API listening on port " + global.settings.node_port); diff --git a/config/environments/development.js b/config/environments/development.js.example similarity index 86% rename from config/environments/development.js rename to config/environments/development.js.example index e18f3a33..17a81cbd 100644 --- a/config/environments/development.js +++ b/config/environments/development.js.example @@ -1,4 +1,5 @@ module.exports.node_port = 8080; +module.exports.node_host = '127.0.0.1'; module.exports.environment = 'development'; module.exports.db_base_name = 'cartodb_dev_user_<%= user_id %>_db'; module.exports.db_user = 'development_cartodb_user_<%= user_id %>'; @@ -9,4 +10,4 @@ module.exports.redis_port = 6379; module.exports.redisPool = 50; module.exports.redisIdleTimeoutMillis = 100; module.exports.redisReapIntervalMillis = 10; -module.exports.redisLog = false; \ No newline at end of file +module.exports.redisLog = false; diff --git a/config/environments/production.js b/config/environments/production.js.example similarity index 100% rename from config/environments/production.js rename to config/environments/production.js.example diff --git a/config/environments/staging.js b/config/environments/staging.js.example similarity index 100% rename from config/environments/staging.js rename to config/environments/staging.js.example diff --git a/config/environments/test.js b/config/environments/test.js.example similarity index 100% rename from config/environments/test.js rename to config/environments/test.js.example diff --git a/configure b/configure new file mode 100755 index 00000000..cbf3e9ee --- /dev/null +++ b/configure @@ -0,0 +1,37 @@ +#!/bin/sh + +usage() { + echo "Usage: $0 [OPTION]" + echo + echo "Configuration:" + echo " --help display this help and exit" + echo " --with-pgport=NUM access PostgreSQL server on TCP port NUM" +} + +PGPORT=5432 + +while test -n "$1"; do + case "$1" in + --help|-h) + usage + exit 0 + ;; + --with-pgport=*) + PGPORT=`echo "$1" | cut -d= -f2` + ;; + *) + echo "Unknown option '$1'" >&2 + usage >&2 + exit 1 + esac + shift +done + +echo "PGPORT: $PGPORT" + +# TODO: allow specifying configuration settings ! +for f in config/environments/*.example; do + o=`dirname "$f"`/`basename "$f" .example` + echo "Writing $o" + sed "s/\( *module.exports.db_port[ \t]*= *'\?\)[^';]*\('\?;\)/\1$PGPORT\2/" < "$f" > "$o" +done diff --git a/doc/API.md b/doc/API.md new file mode 100644 index 00000000..ceb72c09 --- /dev/null +++ b/doc/API.md @@ -0,0 +1,127 @@ +SQL API +======= + +Request format +-------------- + +Supported query string parameters: + + 'q': Specifies the SQL query to run + Example: + 'http://entrypoint?q=SELECT count(*) FROM mytable' + + 'format': Specifies which format to use for the response. + Supported formats: JSON (the default), GeoJSON, + CSV, SVG + + 'dp': Number of digits after the decimal point. + Only affects format=GeoJSON. + By default this is 6. + + 'api_key': Needed to authenticate in order to modify the database. + +Response formats +---------------- + +The standard response from the CartoDB SQL API is JSON. If you are +building a web-application, the lightweight JSON format allows you to +quickly integrate data from the SQL API. + +The JSON response is as follows: +``` + { + time: 0.006, + total_rows: 1, + rows: [ + { + year: " 2011", + the_geom: "0101000020E610...", + cartodb_id: 1, + created_at: "2012-02-06T22:50:35.778Z", + updated_at: "2012-02-12T21:34:08.193Z" + } + ] + } +``` + +Alternatively, you can use the GeoJSON specification for returning data +from the API. To do so, simply supply the format parameter as GeoJSON. + +The GeoJSON response is follows: +``` + { + type: "FeatureCollection", + features: [ + { + type: "Feature", + properties: { + year: " 2011", + month: 10, + day: "11", + cartodb_id: 1, + created_at: "2012-02-06T22:50:35.778Z", + updated_at: "2012-02-12T21:34:08.193Z" + }, + geometry: { + type: "Point", + coordinates: [ + -97.335, + 35.498 + ] + } + } + ] + } +``` + +TODO: csv, kml responses + +Response errors +--------------- + +To help you debug your SQL queries, the CartoDB SQL API returns errors +as part of the JSON response. Errors come back as follows, + +``` + { + error: [ + "syntax error at or near "LIMIT"" + ] + } +``` + +You can use these errors to help understand your SQL. + + +Getting table information +------------------------- + +Currently, there is no public method for accessing your table schemas. The +simplest way to get table structure is to access the first row of the data: + + http://entrypoint?q=SELECT * FROM mytable LIMIT 1 + +Write data to your CartoDB account +---------------------------------- + +Perform inserts or updates on your data is simple now using your API +key. All you need to do, is supply a correct SQL INSERT or UPDATE +statement for your table along with the api_key parameter for your +account. Be sure to keep these requests private, as anyone with your API +key will be able to modify your tables. A correct SQL insert statement +means that all the columns you want to insert into already exist in +your table, and all the values for those columns are the right type +(quoted string, unquoted string for geoms and dates, or numbers). + +INSERT + + http://entrypoint?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={Your API key} + +Updates are just as simple. Here is an example, updating a row based on +the value of the cartodb_id column. + +UPDATE + + http://entrypoint?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={Your API key} + + diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json new file mode 100644 index 00000000..2862185d --- /dev/null +++ b/npm-shrinkwrap.json @@ -0,0 +1,247 @@ +{ + "name": "cartodb_api", + "version": "1.1.0", + "dependencies": { + "cluster2": { + "version": "0.3.5-cdb01", + "from": "git://github.com/CartoDB/cluster2.git#cdb_production", + "dependencies": { + "ejs": { + "version": "0.8.3" + }, + "npm": { + "version": "1.1.62", + "dependencies": { + "semver": { + "version": "1.0.14" + }, + "ini": { + "version": "1.0.4" + }, + "slide": { + "version": "1.1.3" + }, + "abbrev": { + "version": "1.0.3" + }, + "graceful-fs": { + "version": "1.1.14" + }, + "minimatch": { + "version": "0.2.6" + }, + "nopt": { + "version": "2.0.0" + }, + "rimraf": { + "version": "2.0.2" + }, + "request": { + "version": "2.9.203", + "from": "git://github.com/isaacs/request" + }, + "which": { + "version": "1.0.5" + }, + "tar": { + "version": "0.1.13" + }, + "fstream": { + "version": "0.1.19" + }, + "block-stream": { + "version": "0.0.6" + }, + "inherits": { + "version": "1.0.0", + "from": "git://github.com/isaacs/inherits" + }, + "mkdirp": { + "version": "0.3.4" + }, + "read": { + "version": "1.0.4", + "dependencies": { + "mute-stream": { + "version": "0.0.3" + } + } + }, + "lru-cache": { + "version": "2.0.4" + }, + "node-gyp": { + "version": "0.6.11" + }, + "fstream-npm": { + "version": "0.1.2", + "dependencies": { + "fstream-ignore": { + "version": "0.0.5" + } + } + }, + "uid-number": { + "version": "0.0.3" + }, + "archy": { + "version": "0.0.2" + }, + "chownr": { + "version": "0.0.1" + }, + "npmlog": { + "version": "0.0.2" + }, + "ansi": { + "version": "0.1.2" + }, + "npm-registry-client": { + "version": "0.2.7" + }, + "read-package-json": { + "version": "0.1.5" + }, + "read-installed": { + "version": "0.0.2" + }, + "glob": { + "version": "3.1.12" + }, + "init-package-json": { + "version": "0.0.5", + "dependencies": { + "promzard": { + "version": "0.2.0" + } + } + }, + "osenv": { + "version": "0.0.3" + }, + "lockfile": { + "version": "0.2.1" + }, + "retry": { + "version": "0.6.0" + }, + "couch-login": { + "version": "0.1.12" + }, + "once": { + "version": "1.1.1" + }, + "npmconf": { + "version": "0.0.16", + "dependencies": { + "config-chain": { + "version": "1.1.2", + "dependencies": { + "proto-list": { + "version": "1.2.2" + } + } + } + } + }, + "opener": { + "version": "1.3.0" + } + } + } + } + }, + "express": { + "version": "2.5.11", + "dependencies": { + "connect": { + "version": "1.9.2", + "dependencies": { + "formidable": { + "version": "1.0.11" + } + } + }, + "mime": { + "version": "1.2.4" + }, + "qs": { + "version": "0.4.2" + }, + "mkdirp": { + "version": "0.3.0" + } + } + }, + "underscore": { + "version": "1.1.7" + }, + "underscore.string": { + "version": "1.1.5", + "dependencies": { + "underscore": { + "version": "1.1.6" + } + } + }, + "pg": { + "version": "0.6.14", + "dependencies": { + "generic-pool": { + "version": "1.0.9" + } + } + }, + "generic-pool": { + "version": "1.0.12" + }, + "redis": { + "version": "0.7.1" + }, + "hiredis": { + "version": "0.1.14" + }, + "step": { + "version": "0.0.5" + }, + "oauth-client": { + "version": "0.2.0", + "dependencies": { + "node-uuid": { + "version": "1.1.0" + } + } + }, + "node-uuid": { + "version": "1.3.3" + }, + "csv": { + "version": "0.0.13" + }, + "mocha": { + "version": "1.2.1", + "dependencies": { + "commander": { + "version": "0.6.1" + }, + "growl": { + "version": "1.5.1" + }, + "jade": { + "version": "0.26.3", + "dependencies": { + "mkdirp": { + "version": "0.3.0" + } + } + }, + "diff": { + "version": "1.0.2" + }, + "debug": { + "version": "0.7.0" + } + } + } + } +} diff --git a/package.json b/package.json index 7d795be1..fc18f7ed 100644 --- a/package.json +++ b/package.json @@ -2,14 +2,14 @@ "private": true, "name": "cartodb_api", "description": "high speed SQL api for cartodb", - "version": "1.0.0", + "version": "1.1.0", "author": { - "name": "Simon Tokumine, Vizzuality", + "name": "Simon Tokumine, Sandro Santilli, Vizzuality", "url": "http://vizzuality.com", - "email": "simon@vizzuality.com" + "email": "simon@vizzuality.com, strk@vizzuality.com" }, "dependencies": { - "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", + "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "express": "~2.5.11", "underscore" : "1.1.x", "underscore.string": "1.1.5", diff --git a/test/acceptance/app.test.js b/test/acceptance/app.test.js index bf26958d..6dace8a6 100644 --- a/test/acceptance/app.test.js +++ b/test/acceptance/app.test.js @@ -274,6 +274,82 @@ test('GET /api/v1/sql with SQL parameter and geojson format, ensuring content-di }); }); +test('GET /api/v1/sql with SVG format', function(done){ + var query = querystring.stringify({ + q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ", + format: "svg" + }); + assert.response(app, { + url: '/api/v1/sql?' + query, + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('') > 0, res.body ); + // TODO: test viewBox + done(); + }); +}); + +test('GET /api/v1/sql with SVG format and centered point', function(done){ + var query = querystring.stringify({ + q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ", + format: "svg" + }); + assert.response(app, { + url: '/api/v1/sql?' + query, + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('cx="0" cy="0"') > 0, res.body ); + // TODO: test viewBox + // TODO: test radius + done(); + }); +}); + +test('GET /api/v1/sql with SVG format and trimmed decimals', function(done){ + var queryobj = { + q: "SELECT 1 as cartodb_id, 'LINESTRING(0 0, 1024 768, 500.123456 600.98765432)'::geometry AS the_geom ", + format: "svg", + dp: 2 + }; + assert.response(app, { + url: '/api/v1/sql?' + querystring.stringify(queryobj), + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('') > 0, res.body ); + // TODO: test viewBox + + queryobj.dp = 3; + assert.response(app, { + url: '/api/v1/sql?' + querystring.stringify(queryobj), + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('') > 0, res.body ); + // TODO: test viewBox + done(); + }); + }); +}); + test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){ assert.response(app, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4', @@ -393,4 +469,18 @@ test('GET decent error if SQL is broken', function(done){ }); }); +// CSV tests +test('CSV format', function(done){ + assert.response(app, { + url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv', + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.equal(true, /filename=cartodb-query.csv/gi.test(cd)); + done(); + }); +}); + }); diff --git a/test/prepare_db.sh b/test/prepare_db.sh index 4bfb07e1..909d7e95 100755 --- a/test/prepare_db.sh +++ b/test/prepare_db.sh @@ -31,7 +31,7 @@ die() { } echo "preparing postgres..." -dropdb ${TEST_DB} 2> /dev/null # error expected if doesn't exist +dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database" psql -f test.sql ${TEST_DB} diff --git a/test/run_tests.sh b/test/run_tests.sh index df864422..512e5389 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -28,7 +28,7 @@ echo "port ${REDIS_PORT}" | redis-server - > test/test.log & PID_REDIS=$! echo "Preparing the environment" -cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -; +cd test; sh prepare_db.sh || die "database preparation failure"; cd -; PATH=node_modules/.bin/:$PATH diff --git a/tools/cdbsql b/tools/cdbsql index d9d0c465..cd54bfcd 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -3,13 +3,25 @@ // Command line tool for CartoDB SQL API // // https://github.com/Vizzuality/CartoDB-SQL-API +// -var http = require('http') +var http = require('http'); + +var nodevers = process.versions.node.split('.'); + +// NOTE: readline is also available in 0.4 but doesn't work +var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8; +//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support'); + +var readline = hasReadline ? require('readline') : null; var me = process.argv[1]; function usage(exit_code) { console.log("Usage: " + me + " [OPTIONS] "); + if ( hasReadline ) { + console.log(" " + me + " [OPTIONS]"); + } console.log("Options:"); console.log(" -v verbose operations (off)"); console.log(" --help print this help"); @@ -18,18 +30,26 @@ function usage(exit_code) { console.log(" --port service tcp port number (8080)"); console.log(" --api-version API version (1)"); console.log(" --key API authentication key (none)"); + console.log(" --format Response format (json)"); + console.log(" --dp Decimal places in geojson format (unspecified)"); + if ( hasReadline ) { + console.log(" --batch Send all read queries at once (off)"); + } process.exit(exit_code); } process.argv.shift(); // this will be "node" (argv[0]) process.argv.shift(); // this will be "benchmark.js" (argv[1]) +var batch_mode = false; +var format = 'json'; var username; var domain = 'localhost'; var port = 8080; var api_version = 1; var api_key; var sql; +var decimal_places; var arg; while ( arg = process.argv.shift() ) { @@ -54,6 +74,15 @@ while ( arg = process.argv.shift() ) { else if ( arg == '--api-version' ) { api_version = process.argv.shift(); } + else if ( arg == '--format' ) { + format = process.argv.shift(); + } + else if ( arg == '--dp' ) { + decimal_places = process.argv.shift(); + } + else if ( arg == '--batch' ) { + batch_mode = true; + } else if ( ! sql ) { sql = arg; } @@ -62,36 +91,106 @@ while ( arg = process.argv.shift() ) { } } -if ( ! sql ) usage(1); - var hostname = username + '.' + domain; +if ( ! sql ) { + if ( readline ) { + + var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + if ( ! batch_mode ) { + rl.setPrompt(hostname + '> '); + rl.prompt(); + } + + sql = ''; + rl.on('line', function(line) { + sql += line; + if ( ! batch_mode ) { + // TODO: some sanity checking, like trim the line or check if it ends with semicolon + if ( sql.length ) { + processQuery(sql, function() { + sql = ''; + rl.prompt(); + }); + } else rl.prompt(); + } + }).on('close', function() { + if ( batch_mode ) { + if ( sql.length ) { + processQuery(sql); + sql = ''; + } + } else { + if ( sql.length ) { + console.warn("Unprocessed sql left: [" + sql + "]"); + } + console.log("Good bye"); + } + }).on('SIGCONT', function() { + // this is needed so not to exit on stop/resume + rl.prompt(); + }); + } else { + usage(1); + } +} else { + processQuery(sql); +} + // -- Perform the request -var opt = { - host: hostname, - port: port, - path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) -}; +function processQuery(sql, callback) +{ -console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path); + var post_data = 'q=' + encodeURIComponent(sql); -var body = ''; + var opt = { + host: hostname, + port: port, + path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format), + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': post_data.length + } + }; -http.get(opt, function(res) { - console.log("Response status: " + res.statusCode); - res.on('data', function(chunk) { - body += chunk; - //console.log("data: "); console.dir(json); + if ( typeof(api_key) != 'undefined' ) opt.path += '&api_key=' + api_key; + if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places; + + var body = ''; + var request = 'http://' + opt.host + ':' + opt.port + opt.path; + //console.log("Sending request:", request); + + var req = http.request(opt, function(res) { + //console.log("Response status: " + res.statusCode); + res.on('data', function(chunk) { + body += chunk; + //console.log("data: "); console.dir(json); + }); + + res.on('end', function() { + console.log("Request:", request); + var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql; + sqlprint = sqlprint.split('\n').join(' '); + console.log("Query:", sqlprint); + console.log("Response status: " + res.statusCode); + console.log('Response body:'); + console.dir(body); + if ( callback ) callback(); + }); + + }).on('error', function(e) { + console.log("Request:", request); + console.log("Error: " + e.message); + if ( callback ) callback(); }); - res.on('end', function() { - console.log('Body:'); - var json = JSON.parse(body); - console.dir(json); - }); -}).on('error', function(e) { - console.log("ERROR: " + e.message); -}); - + req.write(post_data); + req.end(); +}