From 686ff617b8a4285cc4b07db16036ea20a2ad38b7 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Wed, 3 Oct 2012 16:53:17 +0200 Subject: [PATCH 01/29] updated version to 1.0.0 --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e9daaea1..b47bed08 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +1.0.0 (03/10/12) +----- +* Migrated to node 0.8 version + 0.9.0 (18/09/12) ----- * Fix INSERT and UPDATE with RETURNING clause diff --git a/package.json b/package.json index 047f8576..47e6f1ad 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "private": true, "name": "cartodb_api", "description": "high speed SQL api for cartodb", - "version": "0.9.1-dev", + "version": "1.0.0", "author": { "name": "Simon Tokumine, Vizzuality", "url": "http://vizzuality.com", From b1a06b208ffec6c5fb37554f83818b8aca268778 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Wed, 3 Oct 2012 17:12:31 +0200 Subject: [PATCH 02/29] added staging env to cluster.js --- cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.js b/cluster.js index 5dd3305d..a9c569cb 100755 --- a/cluster.js +++ b/cluster.js @@ -16,7 +16,7 @@ var Cluster = require('cluster2'); var ENV = process.argv[2]; if (ENV != 'development' && ENV != 'production') { console.error("\n./cluster [environment]"); - console.error("environments: [development, test, production]"); + console.error("environments: [development, test, production, staging]"); process.exit(1); } From 3287ffd4e4bd9ec8a0c4bccdd46c9b7fbfbe3a7e Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Wed, 3 Oct 2012 17:14:05 +0200 Subject: [PATCH 03/29] staging env in cluster.js --- cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.js b/cluster.js index a9c569cb..bc705630 100755 --- a/cluster.js +++ b/cluster.js @@ -14,7 +14,7 @@ var Cluster = require('cluster2'); // sanity check arguments var ENV = process.argv[2]; -if (ENV != 'development' && ENV != 'production') { +if (ENV != 'development' && ENV != 'production' && ENV != 'staging') { console.error("\n./cluster [environment]"); console.error("environments: [development, test, production, staging]"); process.exit(1); From 4ec9bf34c15e2b4edfba6fce5998e374a0f796ce Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Thu, 4 Oct 2012 10:54:09 +0200 Subject: [PATCH 04/29] fixed problem in cluster2 with pidfile name --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 47e6f1ad..7d795be1 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "email": "simon@vizzuality.com" }, "dependencies": { - "cluster2": "~0.3.1", + "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", "express": "~2.5.11", "underscore" : "1.1.x", "underscore.string": "1.1.5", From 954e54c87dc46e5c97b8500f3af8a0d923ca6f7e Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Oct 2012 08:38:44 +0200 Subject: [PATCH 05/29] Prepare for 1.1.0 --- NEWS.md | 3 +++ package.json | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index b47bed08..392318c5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +1.1.0 (DD/MM/YY) +----- + 1.0.0 (03/10/12) ----- * Migrated to node 0.8 version diff --git a/package.json b/package.json index 7d795be1..c09f3a2c 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,11 @@ "private": true, "name": "cartodb_api", "description": "high speed SQL api for cartodb", - "version": "1.0.0", + "version": "1.1.0", "author": { - "name": "Simon Tokumine, Vizzuality", + "name": "Simon Tokumine, Sandro Santilli, Vizzuality", "url": "http://vizzuality.com", - "email": "simon@vizzuality.com" + "email": "simon@vizzuality.com, strk@vizzuality.com" }, "dependencies": { "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", From 233f5ecc4b99045b00c77c13d2431cceb494c780 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 23 Jul 2012 19:57:38 +0200 Subject: [PATCH 06/29] Rename env files to .example, add a ./configure script Update README to mention the two possible ways to configure the app --- .gitignore | 3 +- README.md | 9 +++-- ...{development.js => development.js.example} | 0 .../{production.js => production.js.example} | 0 .../{staging.js => staging.js.example} | 0 .../environments/{test.js => test.js.example} | 0 configure | 37 +++++++++++++++++++ 7 files changed, 45 insertions(+), 4 deletions(-) rename config/environments/{development.js => development.js.example} (100%) rename config/environments/{production.js => production.js.example} (100%) rename config/environments/{staging.js => staging.js.example} (100%) rename config/environments/{test.js => test.js.example} (100%) create mode 100755 configure diff --git a/.gitignore b/.gitignore index ae7c93a0..f38ce45e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ +config/environments/*.js logs/*.log pids/*.pid *.sock test/tmp/* node_modules/ -.idea/* \ No newline at end of file +.idea/* diff --git a/README.md b/README.md index 568ed750..b0b43879 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,10 @@ core requirements usage ----- -Edit config/environments/.js +Create and edit config/environments/.js from .js.example files. +You may find the ./configure script useful to make an edited copy for you, +see ```./configure --help``` for a list of supported switches. + Make sure redis is running and knows about active cartodb user. ``` bash @@ -27,7 +30,7 @@ node [cluster.js|app.js] Supported values are developement, test, production -for examples of use, see /tests +for examples of use, see /test Install dependencies @@ -45,4 +48,4 @@ see test/README.md note on 0.4.x -------------- -output of large result sets is slow under node 0.4. Recommend running under 0.6 where possible. +output of large result sets is slow under node 0.4. Recommend running under 0.6+ where possible. diff --git a/config/environments/development.js b/config/environments/development.js.example similarity index 100% rename from config/environments/development.js rename to config/environments/development.js.example diff --git a/config/environments/production.js b/config/environments/production.js.example similarity index 100% rename from config/environments/production.js rename to config/environments/production.js.example diff --git a/config/environments/staging.js b/config/environments/staging.js.example similarity index 100% rename from config/environments/staging.js rename to config/environments/staging.js.example diff --git a/config/environments/test.js b/config/environments/test.js.example similarity index 100% rename from config/environments/test.js rename to config/environments/test.js.example diff --git a/configure b/configure new file mode 100755 index 00000000..cbf3e9ee --- /dev/null +++ b/configure @@ -0,0 +1,37 @@ +#!/bin/sh + +usage() { + echo "Usage: $0 [OPTION]" + echo + echo "Configuration:" + echo " --help display this help and exit" + echo " --with-pgport=NUM access PostgreSQL server on TCP port NUM" +} + +PGPORT=5432 + +while test -n "$1"; do + case "$1" in + --help|-h) + usage + exit 0 + ;; + --with-pgport=*) + PGPORT=`echo "$1" | cut -d= -f2` + ;; + *) + echo "Unknown option '$1'" >&2 + usage >&2 + exit 1 + esac + shift +done + +echo "PGPORT: $PGPORT" + +# TODO: allow specifying configuration settings ! +for f in config/environments/*.example; do + o=`dirname "$f"`/`basename "$f" .example` + echo "Writing $o" + sed "s/\( *module.exports.db_port[ \t]*= *'\?\)[^';]*\('\?;\)/\1$PGPORT\2/" < "$f" > "$o" +done From ae493dd9141bc756169d35803c481e03dda699e3 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 09:30:25 +0200 Subject: [PATCH 07/29] Move install step before usage one (see #16) --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b0b43879..a67b3f62 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,13 @@ core requirements * node > v0.4.8 && < v0.9.0 * npm +Install dependencies +--------------------- + +```bash +npm install +``` + usage ----- @@ -33,14 +40,6 @@ Supported values are developement, test, production for examples of use, see /test -Install dependencies ---------------------- - -```bash -npm install -``` - - tests ------ see test/README.md From 13b3b2f2ffb9a70078a781387a136aab20daaf9c Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 09:45:08 +0200 Subject: [PATCH 08/29] Don't hide errors during db preparation --- test/prepare_db.sh | 2 +- test/run_tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/prepare_db.sh b/test/prepare_db.sh index 4bfb07e1..909d7e95 100755 --- a/test/prepare_db.sh +++ b/test/prepare_db.sh @@ -31,7 +31,7 @@ die() { } echo "preparing postgres..." -dropdb ${TEST_DB} 2> /dev/null # error expected if doesn't exist +dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database" psql -f test.sql ${TEST_DB} diff --git a/test/run_tests.sh b/test/run_tests.sh index df864422..512e5389 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -28,7 +28,7 @@ echo "port ${REDIS_PORT}" | redis-server - > test/test.log & PID_REDIS=$! echo "Preparing the environment" -cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -; +cd test; sh prepare_db.sh || die "database preparation failure"; cd -; PATH=node_modules/.bin/:$PATH From eb645d298cd563df306591ca19657fb76dfe1c39 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 10:29:47 +0200 Subject: [PATCH 09/29] More embedded comments --- app/controllers/app.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/app.js b/app/controllers/app.js index e87172b0..9e5148f3 100755 --- a/app/controllers/app.js +++ b/app/controllers/app.js @@ -12,6 +12,8 @@ // - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public // // eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table +// +// var express = require('express') , app = express.createServer( express.logger({ @@ -47,7 +49,7 @@ function handleQuery(req, res) { var limit = parseInt(req.query.rows_per_page); var offset = parseInt(req.query.page); var format = req.query.format; - var dp = req.query.dp; + var dp = req.query.dp; // decimal point digits (defaults to 6) // sanitize and apply defaults to input dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp; From ba6cde96792b48c8d661e9b9c4a5fee26fd2a4d8 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 10:30:57 +0200 Subject: [PATCH 10/29] Add API documentation to the repository (closes #42) --- README.md | 6 +- doc/API.md | 191 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 doc/API.md diff --git a/README.md b/README.md index a67b3f62..b8d8f9b5 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,14 @@ node [cluster.js|app.js] Supported values are developement, test, production -for examples of use, see /test +See /doc for API documentation. +For examples of use, see /test. tests ------ -see test/README.md + +Run ```make check``` or see test/README.md note on 0.4.x diff --git a/doc/API.md b/doc/API.md new file mode 100644 index 00000000..9fca5f8e --- /dev/null +++ b/doc/API.md @@ -0,0 +1,191 @@ +SQL API +======= + +CartoDB is based on the rock solid PostgreSQL database. All your tables +are inside a single database which means you can perform complex queries +joining tables or performing complicated geospatial operations. The best +place to learn about PostgreSQL SQL language is the official documentation + +CartoDB is also based on PostGIS, so take a look at the official PostGIS +reference to know what functionality we support in terms on geospatial +operations. All our tables include a column called the_geom with the +geometry field and indexes on them in the EPSG:4326 projection. All tables +also have an automatically generated and updated the_geom_webmercator +column that we use internally to create tiles for maps as fast as +possible. + +URL endpoints +------------- + +All SQL API request to your CaroDB account use this pattern + +Pattern + + http://{account}.cartodb.com/api/v2/sql?q={SQL statement} + +Be sure you account name is right and that your SQL statement is valid. A +good test is a simple count of all the records in a table, + +count + + http://{account}.cartodb.com/api/v2/sql?q=SELECT count(*) FROM {table_name} + +result + +``` + { + time: 0.007, + total_rows: 1, + rows: [ + { + count: 4994 + } + ] + } +``` + +Finally, remember that unless you are authenticated your table needs to +be public for the SQL API to work. + +POST and GET +------------ + +The CartoDB SQL API is setup to handle both GET and POST requests. You +can test the GET method directly in your browser. Below is an example +of a JQuery SQL API request to a CartoDB. + +```jQuery + $.getJSON('http://'+your_account_name+'.cartodb.com/api/v2/sql/?q='+sql_statement, function(data){ + $.each(data.rows, function(key, val) { + // do something! + }); + }); +``` + +By default GET requests work from anywhere. In CartoDB, POST requests +work from any website as well. We achieve this by hosting a cross +domain policy file at the root of all of our servers. This allows you +the greatest level of flexibility when developing your application. + +Response formats +---------------- + +The standard response from the CartoDB SQL API is JSON. If you are +building a web-application, the lightweight JSON format allows you to +quickly integrate data from the SQL API. + +example + + http://{account}.cartodb.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 + +result + +``` + { + time: 0.006, + total_rows: 1, + rows: [ + { + year: " 2011", + month: 10, + day: "11", + the_geom: "0101000020E610...", + cartodb_id: 1, + created_at: "2012-02-06T22:50:35.778Z", + updated_at: "2012-02-12T21:34:08.193Z", + the_geom_webmercator: "0101000020110F000..." + } + ] + } +``` + +Alternatively, you can use the GeoJSON specification for returning data +from the API. To do so, simply supply the format parameter as GeoJSON, + +GeoJSON + + http://{account}.cartodb.com/api/v2/sql?format=GeoJSON&q=SELECT * FROM {table_name} LIMIT 1 + +result + +``` + { + type: "FeatureCollection", + features: [ + { + type: "Feature", + properties: { + year: " 2011", + month: 10, + day: "11", + cartodb_id: 1, + created_at: "2012-02-06T22:50:35.778Z", + updated_at: "2012-02-12T21:34:08.193Z" + }, + geometry: { + type: "Point", + coordinates: [ + -97.335, + 35.498 + ] + } + } + ] + } +``` + +Getting table information +------------------------- + +Currently, there is no public method for accessing your table schemas. The +simplest way to get table structure is to access the first row of +the data, + +Columns + + http://{account}.cartodb.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 + +Response errors +--------------- + +To help you debug your SQL queries, the CartoDB SQL API returns errors +as part of the JSON response. Errors come back as follows, + +Errors + +``` + { + error: [ + "syntax error at or near "LIMIT"" + ] + } +``` + +You can use these errors to help understand your SQL, for more complete +documentation see the Error codes and Solutions section of this Users +Guide. + +Write data to your CartoDB account +---------------------------------- + +Perform inserts or updates on your data is simple now using your API +key. All you need to do, is supply a correct SQL INSERT or UPDATE +statement for your table along with the api_key parameter for your +account. Be sure to keep these requests private, as anyone with your API +key will be able to modify your tables. A correct SQL insert statement +means that all the columns you want to insert into already exist in +your table, and all the values for those columns are the right type +(quoted string, unquoted string for geoms and dates, or numbers). + +INSERT + + http://{account}.cartodb.com/api/v2/sql?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={Your API key} + +Updates are just as simple. Here is an example, updating a row based on +the value of the cartodb_id column. + +UPDATE + + http://{account}.cartodb.com/api/v2/sql?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={Your API key} + + From 16c67554a2c1d0f87e66375a867daf4b23232523 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 10:58:26 +0200 Subject: [PATCH 11/29] Generalize the API doc, add 'gn' to the list of supported params --- doc/API.md | 122 +++++++++++++++-------------------------------------- 1 file changed, 34 insertions(+), 88 deletions(-) diff --git a/doc/API.md b/doc/API.md index 9fca5f8e..cc6f44ca 100644 --- a/doc/API.md +++ b/doc/API.md @@ -1,71 +1,28 @@ SQL API ======= -CartoDB is based on the rock solid PostgreSQL database. All your tables -are inside a single database which means you can perform complex queries -joining tables or performing complicated geospatial operations. The best -place to learn about PostgreSQL SQL language is the official documentation +Request format +-------------- -CartoDB is also based on PostGIS, so take a look at the official PostGIS -reference to know what functionality we support in terms on geospatial -operations. All our tables include a column called the_geom with the -geometry field and indexes on them in the EPSG:4326 projection. All tables -also have an automatically generated and updated the_geom_webmercator -column that we use internally to create tiles for maps as fast as -possible. +Supported query string parameters: -URL endpoints -------------- + 'q': Specifies the SQL query to run + Example: + 'http://entrypoint?q=SELECT count(*) FROM mytable' -All SQL API request to your CaroDB account use this pattern + 'format': Specifies which format to use for the response. + Supported formats: JSON (the default), GeoJSON, + csv, kml -Pattern + 'dp': Number of digits after the decimal point. + Only affects format=GeoJSON. + By default this is 6. - http://{account}.cartodb.com/api/v2/sql?q={SQL statement} + 'gn': Geometry column name. + Only affects format=GeoJSON. + By default this is 'the_geom'. -Be sure you account name is right and that your SQL statement is valid. A -good test is a simple count of all the records in a table, - -count - - http://{account}.cartodb.com/api/v2/sql?q=SELECT count(*) FROM {table_name} - -result - -``` - { - time: 0.007, - total_rows: 1, - rows: [ - { - count: 4994 - } - ] - } -``` - -Finally, remember that unless you are authenticated your table needs to -be public for the SQL API to work. - -POST and GET ------------- - -The CartoDB SQL API is setup to handle both GET and POST requests. You -can test the GET method directly in your browser. Below is an example -of a JQuery SQL API request to a CartoDB. - -```jQuery - $.getJSON('http://'+your_account_name+'.cartodb.com/api/v2/sql/?q='+sql_statement, function(data){ - $.each(data.rows, function(key, val) { - // do something! - }); - }); -``` - -By default GET requests work from anywhere. In CartoDB, POST requests -work from any website as well. We achieve this by hosting a cross -domain policy file at the root of all of our servers. This allows you -the greatest level of flexibility when developing your application. + 'api_key': Needed to authenticate in order to modify the database. Response formats ---------------- @@ -74,12 +31,7 @@ The standard response from the CartoDB SQL API is JSON. If you are building a web-application, the lightweight JSON format allows you to quickly integrate data from the SQL API. -example - - http://{account}.cartodb.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 - -result - +The JSON response is as follows: ``` { time: 0.006, @@ -87,27 +39,19 @@ result rows: [ { year: " 2011", - month: 10, - day: "11", the_geom: "0101000020E610...", cartodb_id: 1, created_at: "2012-02-06T22:50:35.778Z", - updated_at: "2012-02-12T21:34:08.193Z", - the_geom_webmercator: "0101000020110F000..." + updated_at: "2012-02-12T21:34:08.193Z" } ] } ``` Alternatively, you can use the GeoJSON specification for returning data -from the API. To do so, simply supply the format parameter as GeoJSON, - -GeoJSON - - http://{account}.cartodb.com/api/v2/sql?format=GeoJSON&q=SELECT * FROM {table_name} LIMIT 1 - -result +from the API. To do so, simply supply the format parameter as GeoJSON. +The GeoJSON response is follows: ``` { type: "FeatureCollection", @@ -134,16 +78,7 @@ result } ``` -Getting table information -------------------------- - -Currently, there is no public method for accessing your table schemas. The -simplest way to get table structure is to access the first row of -the data, - -Columns - - http://{account}.cartodb.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 +TODO: csv, kml responses Response errors --------------- @@ -165,6 +100,17 @@ You can use these errors to help understand your SQL, for more complete documentation see the Error codes and Solutions section of this Users Guide. + +Getting table information +------------------------- + +Currently, there is no public method for accessing your table schemas. The +simplest way to get table structure is to access the first row of the data, + +Columns + + http://entrypoint?q=SELECT * FROM mytable LIMIT 1 + Write data to your CartoDB account ---------------------------------- @@ -179,13 +125,13 @@ your table, and all the values for those columns are the right type INSERT - http://{account}.cartodb.com/api/v2/sql?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={Your API key} + http://entrypoint?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={Your API key} Updates are just as simple. Here is an example, updating a row based on the value of the cartodb_id column. UPDATE - http://{account}.cartodb.com/api/v2/sql?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={Your API key} + http://entrypoint?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={Your API key} From 924905680a8fdbb1a3973a7c0995b309b2a3ec2f Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 11:03:32 +0200 Subject: [PATCH 12/29] Cleanup --- doc/API.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/doc/API.md b/doc/API.md index cc6f44ca..c7fa4ea7 100644 --- a/doc/API.md +++ b/doc/API.md @@ -86,8 +86,6 @@ Response errors To help you debug your SQL queries, the CartoDB SQL API returns errors as part of the JSON response. Errors come back as follows, -Errors - ``` { error: [ @@ -96,18 +94,14 @@ Errors } ``` -You can use these errors to help understand your SQL, for more complete -documentation see the Error codes and Solutions section of this Users -Guide. +You can use these errors to help understand your SQL. Getting table information ------------------------- Currently, there is no public method for accessing your table schemas. The -simplest way to get table structure is to access the first row of the data, - -Columns +simplest way to get table structure is to access the first row of the data: http://entrypoint?q=SELECT * FROM mytable LIMIT 1 From e8401e5c46d28f00722fb054370aa2ae008383b0 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 11:08:13 +0200 Subject: [PATCH 13/29] Cleanup links to docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8d8f9b5..00353a9d 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ node [cluster.js|app.js] Supported values are developement, test, production -See /doc for API documentation. -For examples of use, see /test. +See doc/API.md for API documentation. +For examples of use, see under test/. tests From 45219428207ddf5b56bcc43ccdacbfcd024102aa Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 11:19:06 +0200 Subject: [PATCH 14/29] Test "CSV" format, drop "KML" from the list of supported formats --- app/controllers/app.js | 1 + doc/API.md | 2 +- test/acceptance/app.test.js | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app/controllers/app.js b/app/controllers/app.js index 9e5148f3..258e141f 100755 --- a/app/controllers/app.js +++ b/app/controllers/app.js @@ -159,6 +159,7 @@ function handleQuery(req, res) { } else if (format === 'csv'){ toCSV(result, res, this); } else { + // TODO: error out if 'format' resolves to an unsupported format ! var end = new Date().getTime(); var json_result = {'time' : (end - start)/1000}; diff --git a/doc/API.md b/doc/API.md index c7fa4ea7..7dc1bdf9 100644 --- a/doc/API.md +++ b/doc/API.md @@ -12,7 +12,7 @@ Supported query string parameters: 'format': Specifies which format to use for the response. Supported formats: JSON (the default), GeoJSON, - csv, kml + csv 'dp': Number of digits after the decimal point. Only affects format=GeoJSON. diff --git a/test/acceptance/app.test.js b/test/acceptance/app.test.js index bf26958d..46a2f1ca 100644 --- a/test/acceptance/app.test.js +++ b/test/acceptance/app.test.js @@ -393,4 +393,18 @@ test('GET decent error if SQL is broken', function(done){ }); }); +// CSV tests +test('CSV format', function(done){ + assert.response(app, { + url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv', + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.equal(true, /filename=cartodb-query.csv/gi.test(cd)); + done(); + }); +}); + }); From 31d19d22327449ee2a343325dadb410515307e55 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 24 Jul 2012 11:30:42 +0200 Subject: [PATCH 15/29] Add more info about PostgreSQL requirements -- closes #16 --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 00353a9d..ed86eeb1 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,11 @@ see ```./configure --help``` for a list of supported switches. Make sure redis is running and knows about active cartodb user. +Make sure your PostgreSQL server is running, is accessible on +the host and port specified in the file, has +a 'publicuser' role and trusts user authentication from localhost +connections. + ``` bash node [cluster.js|app.js] ``` From c3906afa09249c081b36cf750dc4ff40cc5f0d7c Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Oct 2012 08:55:06 +0200 Subject: [PATCH 16/29] Drop documentation for unexistent 'gn' parameter --- doc/API.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/API.md b/doc/API.md index 7dc1bdf9..880e01cb 100644 --- a/doc/API.md +++ b/doc/API.md @@ -18,10 +18,6 @@ Supported query string parameters: Only affects format=GeoJSON. By default this is 6. - 'gn': Geometry column name. - Only affects format=GeoJSON. - By default this is 'the_geom'. - 'api_key': Needed to authenticate in order to modify the database. Response formats From 12003799402ee6b04b38ce43a5cee450707c4d28 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 8 Aug 2012 12:18:04 +0200 Subject: [PATCH 17/29] Add --format switch, write response body without parsing it --- tools/cdbsql | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/cdbsql b/tools/cdbsql index d9d0c465..ebbe3a86 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -18,12 +18,14 @@ function usage(exit_code) { console.log(" --port service tcp port number (8080)"); console.log(" --api-version API version (1)"); console.log(" --key API authentication key (none)"); + console.log(" --format Response format (json)"); process.exit(exit_code); } process.argv.shift(); // this will be "node" (argv[0]) process.argv.shift(); // this will be "benchmark.js" (argv[1]) +var format = 'json'; var username; var domain = 'localhost'; var port = 8080; @@ -54,6 +56,9 @@ while ( arg = process.argv.shift() ) { else if ( arg == '--api-version' ) { api_version = process.argv.shift(); } + else if ( arg == '--format' ) { + format = process.argv.shift(); + } else if ( ! sql ) { sql = arg; } @@ -71,7 +76,7 @@ var hostname = username + '.' + domain; var opt = { host: hostname, port: port, - path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) + path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) + '&format=' + encodeURIComponent(format) }; console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path); @@ -87,8 +92,7 @@ http.get(opt, function(res) { res.on('end', function() { console.log('Body:'); - var json = JSON.parse(body); - console.dir(json); + console.dir(body); }); }).on('error', function(e) { console.log("ERROR: " + e.message); From be2bc87bb6dac85a5bfa285464c1865a42c1f810 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 8 Aug 2012 12:27:57 +0200 Subject: [PATCH 18/29] Add --dp switch to specify decimal places for geojson output --- tools/cdbsql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/cdbsql b/tools/cdbsql index ebbe3a86..c0c9881e 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -19,6 +19,7 @@ function usage(exit_code) { console.log(" --api-version API version (1)"); console.log(" --key API authentication key (none)"); console.log(" --format Response format (json)"); + console.log(" --dp Decimal places in geojson format (unspecified)"); process.exit(exit_code); } @@ -32,6 +33,7 @@ var port = 8080; var api_version = 1; var api_key; var sql; +var decimal_places; var arg; while ( arg = process.argv.shift() ) { @@ -59,6 +61,9 @@ while ( arg = process.argv.shift() ) { else if ( arg == '--format' ) { format = process.argv.shift(); } + else if ( arg == '--dp' ) { + decimal_places = process.argv.shift(); + } else if ( ! sql ) { sql = arg; } @@ -79,6 +84,8 @@ var opt = { path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) + '&format=' + encodeURIComponent(format) }; +if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places; + console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path); var body = ''; From 8078bd55e70954fc275cbcb5454bf7e98f7d268c Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 8 Aug 2012 14:28:07 +0200 Subject: [PATCH 19/29] Add support for reading queries from stdin Only available with node-0.8+ --- tools/cdbsql | 79 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/tools/cdbsql b/tools/cdbsql index c0c9881e..46da6a0b 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -3,13 +3,25 @@ // Command line tool for CartoDB SQL API // // https://github.com/Vizzuality/CartoDB-SQL-API +// -var http = require('http') +var http = require('http'); + +var nodevers = process.versions.node.split('.'); + +// NOTE: readline is also available in 0.4 but doesn't work +var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8; +//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support'); + +var readline = hasReadline ? require('readline') : null; var me = process.argv[1]; function usage(exit_code) { console.log("Usage: " + me + " [OPTIONS] "); + if ( hasReadline ) { + console.log(" " + me + " [OPTIONS]"); + } console.log("Options:"); console.log(" -v verbose operations (off)"); console.log(" --help print this help"); @@ -72,37 +84,54 @@ while ( arg = process.argv.shift() ) { } } -if ( ! sql ) usage(1); - var hostname = username + '.' + domain; +if ( ! sql ) { + if ( readline ) { + var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + rl.on('line', function(sql) { + // TODO: some sanity checking, like trim the line ? + if ( sql ) processQuery(sql); + }); + } else { + usage(1); + } +} else { + processQuery(sql); +} + // -- Perform the request -var opt = { - host: hostname, - port: port, - path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) + '&format=' + encodeURIComponent(format) -}; +function processQuery(sql) +{ + var opt = { + host: hostname, + port: port, + path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) + '&format=' + encodeURIComponent(format) + }; -if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places; + if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places; -console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path); + console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path); -var body = ''; + var body = ''; -http.get(opt, function(res) { - console.log("Response status: " + res.statusCode); - res.on('data', function(chunk) { - body += chunk; - //console.log("data: "); console.dir(json); + http.get(opt, function(res) { + console.log("Response status: " + res.statusCode); + res.on('data', function(chunk) { + body += chunk; + //console.log("data: "); console.dir(json); + }); + + res.on('end', function() { + console.log('Body:'); + console.dir(body); + }); + }).on('error', function(e) { + console.log("ERROR: " + e.message); }); - res.on('end', function() { - console.log('Body:'); - console.dir(body); - }); -}).on('error', function(e) { - console.log("ERROR: " + e.message); -}); - - +} From f725cb4f59aac4143338f4a1807c94662556b35c Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 8 Aug 2012 15:02:43 +0200 Subject: [PATCH 20/29] Add prompt and batch mode --- tools/cdbsql | 62 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/tools/cdbsql b/tools/cdbsql index 46da6a0b..a2ec3b4b 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -32,12 +32,16 @@ function usage(exit_code) { console.log(" --key API authentication key (none)"); console.log(" --format Response format (json)"); console.log(" --dp Decimal places in geojson format (unspecified)"); + if ( hasReadline ) { + console.log(" --batch Send all read queries at once (off)"); + } process.exit(exit_code); } process.argv.shift(); // this will be "node" (argv[0]) process.argv.shift(); // this will be "benchmark.js" (argv[1]) +var batch_mode = false; var format = 'json'; var username; var domain = 'localhost'; @@ -76,6 +80,9 @@ while ( arg = process.argv.shift() ) { else if ( arg == '--dp' ) { decimal_places = process.argv.shift(); } + else if ( arg == '--batch' ) { + batch_mode = true; + } else if ( ! sql ) { sql = arg; } @@ -88,13 +95,44 @@ var hostname = username + '.' + domain; if ( ! sql ) { if ( readline ) { + var rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - rl.on('line', function(sql) { - // TODO: some sanity checking, like trim the line ? - if ( sql ) processQuery(sql); + + if ( ! batch_mode ) { + rl.setPrompt(hostname + '> '); + rl.prompt(); + } + + sql = ''; + rl.on('line', function(line) { + sql += line; + if ( ! batch_mode ) { + // TODO: some sanity checking, like trim the line or check if it ends with semicolon + if ( sql.length ) { + processQuery(sql, function() { + sql = ''; + rl.prompt(); + }); + } else rl.prompt(); + } + }).on('close', function() { + if ( batch_mode ) { + if ( sql.length ) { + processQuery(sql); + sql = ''; + } + } else { + if ( sql.length ) { + console.warn("Unprocessed sql left: [" + sql + "]"); + } + console.log("Good bye"); + } + }).on('SIGCONT', function() { + // this is needed so not to exit on stop/resume + rl.prompt(); }); } else { usage(1); @@ -105,7 +143,7 @@ if ( ! sql ) { // -- Perform the request -function processQuery(sql) +function processQuery(sql, callback) { var opt = { host: hostname, @@ -115,23 +153,29 @@ function processQuery(sql) if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places; - console.log("Requests:", 'http://' + opt.host + ':' + opt.port + opt.path); - var body = ''; + var request = 'http://' + opt.host + ':' + opt.port + opt.path; + //console.log("Sending request:", request); http.get(opt, function(res) { - console.log("Response status: " + res.statusCode); + //console.log("Response status: " + res.statusCode); res.on('data', function(chunk) { body += chunk; //console.log("data: "); console.dir(json); }); res.on('end', function() { - console.log('Body:'); + console.log("Request:", request); + console.log("Response status: " + res.statusCode); + console.log('Response body:'); console.dir(body); + if ( callback ) callback(); }); + }).on('error', function(e) { - console.log("ERROR: " + e.message); + console.log("Request:", request); + console.log("Error: " + e.message); + if ( callback ) callback(); }); } From 2161eb1a156cf7bbd4007ff71e250f905d3455b9 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 8 Aug 2012 16:49:15 +0200 Subject: [PATCH 21/29] Fix handling of --key switch --- tools/cdbsql | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/cdbsql b/tools/cdbsql index a2ec3b4b..dfc384a5 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -151,6 +151,7 @@ function processQuery(sql, callback) path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) + '&format=' + encodeURIComponent(format) }; + if ( typeof(api_key) != 'undefined' ) opt.path += '&api_key=' + api_key; if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places; var body = ''; From fee948324a22b2b27a1ca617f351371cbe664ab3 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 8 Aug 2012 17:27:29 +0200 Subject: [PATCH 22/29] Use POST method to send queries (may be very long) --- tools/cdbsql | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tools/cdbsql b/tools/cdbsql index dfc384a5..cd54bfcd 100755 --- a/tools/cdbsql +++ b/tools/cdbsql @@ -145,10 +145,18 @@ if ( ! sql ) { function processQuery(sql, callback) { + + var post_data = 'q=' + encodeURIComponent(sql); + var opt = { host: hostname, port: port, - path: '/api/v' + api_version + '/sql?q=' + encodeURIComponent(sql) + '&format=' + encodeURIComponent(format) + path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format), + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': post_data.length + } }; if ( typeof(api_key) != 'undefined' ) opt.path += '&api_key=' + api_key; @@ -158,7 +166,7 @@ function processQuery(sql, callback) var request = 'http://' + opt.host + ':' + opt.port + opt.path; //console.log("Sending request:", request); - http.get(opt, function(res) { + var req = http.request(opt, function(res) { //console.log("Response status: " + res.statusCode); res.on('data', function(chunk) { body += chunk; @@ -167,6 +175,9 @@ function processQuery(sql, callback) res.on('end', function() { console.log("Request:", request); + var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql; + sqlprint = sqlprint.split('\n').join(' '); + console.log("Query:", sqlprint); console.log("Response status: " + res.statusCode); console.log('Response body:'); console.dir(body); @@ -179,4 +190,7 @@ function processQuery(sql, callback) if ( callback ) callback(); }); + req.write(post_data); + req.end(); + } From 6a2c0e9727c6baf4e7bd918f73eed6637d43006c Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 11 Sep 2012 12:22:27 +0200 Subject: [PATCH 23/29] Initial support for SVG output (#49) This version only dumps the geometries with no identifier. Scales geometries to fit in a 1024x768 pixels area, using a circle radius of 5 pixels, a stroke-width of 1 pixel and no fill. Supports trimming number of decimals. Adds a viewBox tag to fit the drawing to the output device window. Includes an automated testcase. --- app/controllers/app.js | 127 +++++++++++++++++++++++++++++++++++- test/acceptance/app.test.js | 76 +++++++++++++++++++++ 2 files changed, 202 insertions(+), 1 deletion(-) diff --git a/app/controllers/app.js b/app/controllers/app.js index 258e141f..9e9f33bc 100755 --- a/app/controllers/app.js +++ b/app/controllers/app.js @@ -50,6 +50,10 @@ function handleQuery(req, res) { var offset = parseInt(req.query.page); var format = req.query.format; var dp = req.query.dp; // decimal point digits (defaults to 6) + var gn = "the_geom"; // TODO: read from configuration file + var svg_width = 1024.0; + var svg_height = 768.0; + // sanitize and apply defaults to input dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp; @@ -129,6 +133,27 @@ function handleQuery(req, res) { // TODO: refactor formats to external object if (format === 'geojson'){ sql = ['SELECT *, ST_AsGeoJSON(the_geom,',dp,') as the_geom FROM (', sql, ') as foo'].join(""); + } else if (format === 'svg'){ + var svg_ratio = svg_width/svg_height; + sql = 'WITH source AS ( ' + sql + '), extent AS ( ' + + ' SELECT ST_Extent(' + gn + ') AS e FROM source ' + + '), extent_info AS ( SELECT e, ' + + 'st_xmin(e) as ex0, st_ymax(e) as ey0, ' + + 'st_xmax(e)-st_xmin(e) as ew, ' + + 'st_ymax(e)-st_ymin(e) as eh FROM extent )' + + ', trans AS ( SELECT CASE WHEN ' + + 'eh = 0 THEN ' + svg_width + + '/ COALESCE(NULLIF(ew,0),' + svg_width +') WHEN ' + + svg_ratio + ' <= (ew / eh) THEN (' + + svg_width + '/ew ) ELSE (' + + svg_height + '/eh ) END as s ' + + ', ex0 as x0, ey0 as y0 FROM extent_info ) ' + + 'SELECT st_TransScale(e, -x0, -y0, s, s)::box2d as ' + + gn + '_box, ST_Dimension(' + gn + ') as ' + gn + + '_dimension, ST_AsSVG(ST_TransScale(' + gn + ', ' + + '-x0, -y0, s, s), 0, ' + dp + ') as ' + gn + //+ ', ex0, ey0, ew, eh, s ' // DEBUG ONLY + + ' FROM trans, extent_info, source'; } pg.query(sql, this); @@ -156,6 +181,8 @@ function handleQuery(req, res) { // TODO: refactor formats to external object if (format === 'geojson'){ toGeoJSON(result, res, this); + } else if (format === 'svg'){ + toSVG(result.rows, gn, this); } else if (format === 'csv'){ toCSV(result, res, this); } else { @@ -221,6 +248,98 @@ function toGeoJSON(data, res, callback){ } } +function toSVG(rows, gn, callback){ + + var radius = 5; // in pixels (based on svg_width and svg_height) + var stroke_width = 1; // in pixels (based on svg_width and svg_height) + var stroke_color = 'black'; + // fill settings affect polygons and points (circles) + var fill_opacity = 0.5; // 0.0 is fully transparent, 1.0 is fully opaque + // unused if fill_color='none' + var fill_color = 'none'; // affects polygons and circles + + var bbox; // will be computed during the results scan + var polys = []; + var lines = []; + var points = []; + _.each(rows, function(ele){ + var g = ele[gn]; + if ( ! g ) return; // null or empty + var gdims = ele[gn + '_dimension']; + + // TODO: add an identifier, if any of "cartodb_id", "oid", "id", "gid" are found + // TODO: add "class" attribute to help with styling ? + if ( gdims == '0' ) { + points.push(''); + } else if ( gdims == '1' ) { + // Avoid filling closed linestrings + var linetag = ''; + lines.push(linetag); + } else if ( gdims == '2' ) { + polys.push(''); + } + + if ( ! bbox ) { + // Parse layer extent: "BOX(x y, X Y)" + // NOTE: the name of the extent field is + // determined by the same code adding the + // ST_AsSVG call (in queryResult) + // + bbox = ele[gn + '_box']; + bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/); + bbox = { + xmin: parseFloat(bbox[1]), + ymin: parseFloat(bbox[2]), + xmax: parseFloat(bbox[3]), + ymax: parseFloat(bbox[4]) + }; + } + }); + + // Set point radius + for (var i=0; i', + '', + ]; + + var root_tag = ''); + + // return payload + callback(null, out.join("\n")); +} + function toCSV(data, res, callback){ try{ // pull out keys for column headers @@ -241,9 +360,12 @@ function getContentDisposition(format){ if (format === 'geojson'){ ext = 'geojson'; } - if (format === 'csv'){ + else if (format === 'csv'){ ext = 'csv'; } + else if (format === 'svg'){ + ext = 'svg'; + } var time = new Date().toUTCString(); return 'inline; filename=cartodb-query.' + ext + '; modification-date="' + time + '";'; } @@ -253,6 +375,9 @@ function getContentType(format){ if (format === 'csv'){ type = "text/csv; charset=utf-8"; } + else if (format === 'svg'){ + type = "image/svg+xml; charset=utf-8"; + } return type; } diff --git a/test/acceptance/app.test.js b/test/acceptance/app.test.js index 46a2f1ca..6dace8a6 100644 --- a/test/acceptance/app.test.js +++ b/test/acceptance/app.test.js @@ -274,6 +274,82 @@ test('GET /api/v1/sql with SQL parameter and geojson format, ensuring content-di }); }); +test('GET /api/v1/sql with SVG format', function(done){ + var query = querystring.stringify({ + q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ", + format: "svg" + }); + assert.response(app, { + url: '/api/v1/sql?' + query, + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('') > 0, res.body ); + // TODO: test viewBox + done(); + }); +}); + +test('GET /api/v1/sql with SVG format and centered point', function(done){ + var query = querystring.stringify({ + q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ", + format: "svg" + }); + assert.response(app, { + url: '/api/v1/sql?' + query, + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('cx="0" cy="0"') > 0, res.body ); + // TODO: test viewBox + // TODO: test radius + done(); + }); +}); + +test('GET /api/v1/sql with SVG format and trimmed decimals', function(done){ + var queryobj = { + q: "SELECT 1 as cartodb_id, 'LINESTRING(0 0, 1024 768, 500.123456 600.98765432)'::geometry AS the_geom ", + format: "svg", + dp: 2 + }; + assert.response(app, { + url: '/api/v1/sql?' + querystring.stringify(queryobj), + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{ }, function(res){ + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('') > 0, res.body ); + // TODO: test viewBox + + queryobj.dp = 3; + assert.response(app, { + url: '/api/v1/sql?' + querystring.stringify(queryobj), + headers: {host: 'vizzuality.cartodb.com'}, + method: 'GET' + },{}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var cd = res.header('Content-Disposition'); + assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); + assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.ok( res.body.indexOf('') > 0, res.body ); + // TODO: test viewBox + done(); + }); + }); +}); + test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){ assert.response(app, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4', From a3357b6841b3a30760aa27964cd30ae61b2fea5f Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Oct 2012 12:03:34 +0200 Subject: [PATCH 24/29] Add info about what's changed so far --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index 392318c5..183bb98c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,12 @@ 1.1.0 (DD/MM/YY) ----- +* Fixed problem in cluster2 with pidfile name +* SVG output format +* Enhancement to the cdbsql tool: + - New switches: --format, --key, --dp + - Interactive mode +* API documentation +* ./configure script 1.0.0 (03/10/12) ----- From 9090a191a803e469e237da88afc5aded12c8e294 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Oct 2012 12:43:12 +0200 Subject: [PATCH 25/29] Document node-0.8 requirement --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ed86eeb1..cc36d107 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ Provides a nodejs based API for running SQL queries against CartoDB. core requirements ------------- * postgres 9.0+ -* cartodb 0.9.5+ (for CDB_QueryTables) +* cartodb 0.9.5+ (for ``CDB_QueryTables``) * redis -* node > v0.4.8 && < v0.9.0 +* node 0.8+ * npm Install dependencies @@ -51,7 +51,3 @@ tests Run ```make check``` or see test/README.md - -note on 0.4.x --------------- -output of large result sets is slow under node 0.4. Recommend running under 0.6+ where possible. From aa350768ef4eac105f6fad79a50cc7f7f4e3ecd4 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 18:40:17 +0200 Subject: [PATCH 26/29] Restrict listening host to configured ``node_host`` --- app.js | 5 +++-- cluster.js | 6 ++++-- config/environments/development.js.example | 3 ++- package.json | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app.js b/app.js index 21de716b..58c9cf6b 100755 --- a/app.js +++ b/app.js @@ -26,5 +26,6 @@ _.extend(global.settings, env); // kick off controller var app = require(global.settings.app_root + '/app/controllers/app'); -app.listen(global.settings.node_port); -console.log("CartoDB SQL API listening on port " + global.settings.node_port); \ No newline at end of file +app.listen(global.settings.node_port, global.settings.node_host, function() { + console.log("CartoDB SQL API listening on " + global.settings.node_host + ":" + global.settings.node_port); +}); diff --git a/cluster.js b/cluster.js index bc705630..30417979 100755 --- a/cluster.js +++ b/cluster.js @@ -30,11 +30,13 @@ var app = require(global.settings.app_root + '/app/controllers/app'); var cluster = new Cluster({ port: global.settings.node_port, + host: global.settings.node_host, + monHost: global.settings.node_host, monPort: global.settings.node_port+1 }); cluster.listen(function(cb) { cb(app); +}, function() { + console.log("CartoDB SQL API listening on " + global.settings.node_host + ':' + global.settings.node_port); }); - -console.log("CartoDB SQL API listening on port " + global.settings.node_port); diff --git a/config/environments/development.js.example b/config/environments/development.js.example index e18f3a33..17a81cbd 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -1,4 +1,5 @@ module.exports.node_port = 8080; +module.exports.node_host = '127.0.0.1'; module.exports.environment = 'development'; module.exports.db_base_name = 'cartodb_dev_user_<%= user_id %>_db'; module.exports.db_user = 'development_cartodb_user_<%= user_id %>'; @@ -9,4 +10,4 @@ module.exports.redis_port = 6379; module.exports.redisPool = 50; module.exports.redisIdleTimeoutMillis = 100; module.exports.redisReapIntervalMillis = 10; -module.exports.redisLog = false; \ No newline at end of file +module.exports.redisLog = false; diff --git a/package.json b/package.json index c09f3a2c..fc18f7ed 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "email": "simon@vizzuality.com, strk@vizzuality.com" }, "dependencies": { - "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", + "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "express": "~2.5.11", "underscore" : "1.1.x", "underscore.string": "1.1.5", From e43fdc409a12424c7a50f2a00992cbba0b88ad24 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 18:42:10 +0200 Subject: [PATCH 27/29] Shrinkwrapped current deps --- npm-shrinkwrap.json | 247 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 npm-shrinkwrap.json diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json new file mode 100644 index 00000000..2862185d --- /dev/null +++ b/npm-shrinkwrap.json @@ -0,0 +1,247 @@ +{ + "name": "cartodb_api", + "version": "1.1.0", + "dependencies": { + "cluster2": { + "version": "0.3.5-cdb01", + "from": "git://github.com/CartoDB/cluster2.git#cdb_production", + "dependencies": { + "ejs": { + "version": "0.8.3" + }, + "npm": { + "version": "1.1.62", + "dependencies": { + "semver": { + "version": "1.0.14" + }, + "ini": { + "version": "1.0.4" + }, + "slide": { + "version": "1.1.3" + }, + "abbrev": { + "version": "1.0.3" + }, + "graceful-fs": { + "version": "1.1.14" + }, + "minimatch": { + "version": "0.2.6" + }, + "nopt": { + "version": "2.0.0" + }, + "rimraf": { + "version": "2.0.2" + }, + "request": { + "version": "2.9.203", + "from": "git://github.com/isaacs/request" + }, + "which": { + "version": "1.0.5" + }, + "tar": { + "version": "0.1.13" + }, + "fstream": { + "version": "0.1.19" + }, + "block-stream": { + "version": "0.0.6" + }, + "inherits": { + "version": "1.0.0", + "from": "git://github.com/isaacs/inherits" + }, + "mkdirp": { + "version": "0.3.4" + }, + "read": { + "version": "1.0.4", + "dependencies": { + "mute-stream": { + "version": "0.0.3" + } + } + }, + "lru-cache": { + "version": "2.0.4" + }, + "node-gyp": { + "version": "0.6.11" + }, + "fstream-npm": { + "version": "0.1.2", + "dependencies": { + "fstream-ignore": { + "version": "0.0.5" + } + } + }, + "uid-number": { + "version": "0.0.3" + }, + "archy": { + "version": "0.0.2" + }, + "chownr": { + "version": "0.0.1" + }, + "npmlog": { + "version": "0.0.2" + }, + "ansi": { + "version": "0.1.2" + }, + "npm-registry-client": { + "version": "0.2.7" + }, + "read-package-json": { + "version": "0.1.5" + }, + "read-installed": { + "version": "0.0.2" + }, + "glob": { + "version": "3.1.12" + }, + "init-package-json": { + "version": "0.0.5", + "dependencies": { + "promzard": { + "version": "0.2.0" + } + } + }, + "osenv": { + "version": "0.0.3" + }, + "lockfile": { + "version": "0.2.1" + }, + "retry": { + "version": "0.6.0" + }, + "couch-login": { + "version": "0.1.12" + }, + "once": { + "version": "1.1.1" + }, + "npmconf": { + "version": "0.0.16", + "dependencies": { + "config-chain": { + "version": "1.1.2", + "dependencies": { + "proto-list": { + "version": "1.2.2" + } + } + } + } + }, + "opener": { + "version": "1.3.0" + } + } + } + } + }, + "express": { + "version": "2.5.11", + "dependencies": { + "connect": { + "version": "1.9.2", + "dependencies": { + "formidable": { + "version": "1.0.11" + } + } + }, + "mime": { + "version": "1.2.4" + }, + "qs": { + "version": "0.4.2" + }, + "mkdirp": { + "version": "0.3.0" + } + } + }, + "underscore": { + "version": "1.1.7" + }, + "underscore.string": { + "version": "1.1.5", + "dependencies": { + "underscore": { + "version": "1.1.6" + } + } + }, + "pg": { + "version": "0.6.14", + "dependencies": { + "generic-pool": { + "version": "1.0.9" + } + } + }, + "generic-pool": { + "version": "1.0.12" + }, + "redis": { + "version": "0.7.1" + }, + "hiredis": { + "version": "0.1.14" + }, + "step": { + "version": "0.0.5" + }, + "oauth-client": { + "version": "0.2.0", + "dependencies": { + "node-uuid": { + "version": "1.1.0" + } + } + }, + "node-uuid": { + "version": "1.3.3" + }, + "csv": { + "version": "0.0.13" + }, + "mocha": { + "version": "1.2.1", + "dependencies": { + "commander": { + "version": "0.6.1" + }, + "growl": { + "version": "1.5.1" + }, + "jade": { + "version": "0.26.3", + "dependencies": { + "mkdirp": { + "version": "0.3.0" + } + } + }, + "diff": { + "version": "1.0.2" + }, + "debug": { + "version": "0.7.0" + } + } + } + } +} From 490b3f7e1f6a3cdedfb86df3fccc59b5a936ad0a Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 10 Oct 2012 18:51:47 +0200 Subject: [PATCH 28/29] Add SVG to the list of supported formats --- doc/API.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/API.md b/doc/API.md index 880e01cb..ceb72c09 100644 --- a/doc/API.md +++ b/doc/API.md @@ -12,7 +12,7 @@ Supported query string parameters: 'format': Specifies which format to use for the response. Supported formats: JSON (the default), GeoJSON, - csv + CSV, SVG 'dp': Number of digits after the decimal point. Only affects format=GeoJSON. From 311262c2adc455ae77e9417d2ffc058bf624f732 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Tue, 30 Oct 2012 13:11:34 +0100 Subject: [PATCH 29/29] updated NEWS for 1.1.0 --- NEWS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 183bb98c..1d3e3e93 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -1.1.0 (DD/MM/YY) +1.1.0 (30/10/12) ----- * Fixed problem in cluster2 with pidfile name * SVG output format @@ -7,6 +7,7 @@ - Interactive mode * API documentation * ./configure script +* Restrict listening to a node host 1.0.0 (03/10/12) -----