cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support for specifying a list of fields to skip from output.

Browse Source 
Closes #63
This commit is contained in:
Sandro Santilli 2012-11-12 17:10:16 +01:00
parent ab9c739995
commit 005ae48e3a
6 changed files with 178 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -1,6 +1,7 @@
1.3.0 (DD/MM/YY)
-----
* Support for specifying a filename for exports (#64)
* Support for specifying a list of fields to skip from output (#63)
1.2.1 (DD/MM/YY)
-----

16
app/controllers/app.js
View File

@ -91,6 +91,8 @@ function handleQuery(req, res) {
var offset = parseInt(req.query.page);
var format = _.isArray(req.query.format) ? _.last(req.query.format) : req.query.format;
var filename = req.query.filename;
var skipfields = req.query.skipfields ? req.query.skipfields.split(',') : [];
req.query.skipfields = skipfields; // save back, for toOGR use
var dp = req.query.dp; // decimal point digits (defaults to 6)
var gn = "the_geom"; // TODO: read from configuration file
var user_id;
@ -231,6 +233,14 @@ function handleQuery(req, res) {
function packageResults(err, result){
if (err) throw err;
if ( skipfields.length ){
for ( var i=0; i<result.rows.length; ++i ) {
for ( var j=0; j<skipfields.length; ++j ) {
delete result.rows[i][skipfields[j]];
}
}
}
// TODO: refactor formats to external object
if (format === 'geojson'){
toGeoJSON(result, res, this);
@ -248,7 +258,6 @@ function handleQuery(req, res) {
var json_result = {'time' : (end - start)/1000};
json_result.total_rows = result.rowCount;
json_result.rows = result.rows;
return json_result;
}
else throw new Error("Unexpected format in packageResults: " + format);
@ -423,6 +432,9 @@ function toOGR(dbname, user_id, gcol, sql, res, out_format, out_filename, callba
var tmpdir = '/tmp'; // FIXME: make configurable
var columns = [];
var skipfields = res.req.query.skipfields;
skipfields.push( "the_geom_webmercator" );
Step (
function fetchColumns() {
@ -438,7 +450,7 @@ function toOGR(dbname, user_id, gcol, sql, res, out_format, out_filename, callba
// Skip system columns
for (var k in result.rows[0]) {
if ( k === "the_geom_webmercator" ) continue;
if ( skipfields.indexOf(k) != -1 ) continue;
columns.push('"' + k + '"');
}
//console.log(columns.join(','));

4
doc/API.md
View File

@ -17,6 +17,10 @@ Supported query string parameters:
'filename': Sets the filename to use for the query result
file attachment
'skipfields':
Comma separate list of fields that are not wanted
in output. Only useful with "SELECT *" queries.
'dp': Number of digits after the decimal point.
Only affects format=GeoJSON.
By default this is 6.

10
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "cartodb_api",
"version": "1.3.0-dev",
"version": "1.3.0",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb02",
@ -221,6 +221,14 @@
"zipfile": {
"version": "0.3.2"
},
"libxmljs": {
"version": "0.6.1",
"dependencies": {
"bindings": {
"version": "1.0.0"
}
}
},
"mocha": {
"version": "1.2.1",
"dependencies": {

3
package.json
View File

@ -24,7 +24,8 @@
},
"devDependencies": {
"mocha": "1.2.1",
"zipfile": "~0.3.2"
"zipfile": "~0.3.2",
"libxmljs": "~0.6.1"
},
"scripts": {
"test": "test/run_tests.sh"

149
test/acceptance/app.test.js
View File

@ -21,6 +21,7 @@ var app = require(global.settings.app_root + '/app/controllers/app')
, _ = require('underscore')
, zipfile = require('zipfile')
, fs = require('fs')
, libxmljs = require('libxmljs')
;
// allow lots of emitters to be set to silence warning
@ -431,6 +432,46 @@ test('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposi
});
});
test('field named "the_geom_webmercator" is not skipped by default', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var row0 = JSON.parse(res.body).rows[0];
var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':1, 'the_geom_webmercator':1};
for ( var f in checkfields ) {
if ( checkfields[f] ) {
assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
} else {
assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
}
}
done();
});
});
test('skipfields controls included fields', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&skipfields=the_geom_webmercator,cartodb_id,unexistant',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var row0 = JSON.parse(res.body).rows[0];
var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':0};
for ( var f in checkfields ) {
if ( checkfields[f] ) {
assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
} else {
assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
}
}
done();
});
});
test('GET /api/v1/sql ensure cross domain set on errors', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*gadfgadfg%20FROM%20untitle_table_4',
@ -527,6 +568,48 @@ test('uses custom filename', function(done){
});
});
test('does not include the_geom and the_geom_webmercator properties by default', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var parsed_body = JSON.parse(res.body);
var row0 = parsed_body.features[0].properties;
var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':0, 'the_geom_webmercator':0};
for ( var f in checkfields ) {
if ( checkfields[f] ) {
assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
} else {
assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
}
}
done();
});
});
test('skipfields controls fields included in GeoJSON output', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&skipfields=unexistant,cartodb_id',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var parsed_body = JSON.parse(res.body);
var row0 = parsed_body.features[0].properties;
var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':0, 'the_geom_webmercator':0};
for ( var f in checkfields ) {
if ( checkfields[f] ) {
assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
} else {
assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
}
}
done();
});
});
test('GET /api/v1/sql as geojson limiting decimal places', function(done){
assert.response(app, {
@ -590,6 +673,37 @@ test('CSV format, custom filename', function(done){
assert.equal(true, /filename=mycsv.csv/gi.test(cd), cd);
var ct = res.header('Content-Type');
assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':1, 'the_geom_webmercator':1};
for ( var f in checkfields ) {
var idx = row0.indexOf(f);
if ( checkfields[f] ) {
assert.ok(idx != -1, "result does not include '" + f + "'");
} else {
assert.ok(idx == -1, "result includes '" + f + "' ("+idx+")");
}
}
done();
});
});
test('skipfields controls fields included in CSV output', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv&skipfields=unexistant,cartodb_id',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':1};
for ( var f in checkfields ) {
var idx = row0.indexOf(f);
if ( checkfields[f] ) {
assert.ok(idx != -1, "result does not include '" + f + "'");
} else {
assert.ok(idx == -1, "result includes '" + f + "' ("+idx+")");
}
}
done();
});
});
@ -757,6 +871,7 @@ test('SHP format, unauthenticated', function(done){
assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
// missing SRID, so no PRJ (TODO: add ?)
//assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
// TODO: check DBF contents
fs.unlinkSync(tmpfile);
done();
});
@ -832,6 +947,7 @@ test('SHP format, authenticated', function(done){
assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
// missing SRID, so no PRJ (TODO: add ?)
//assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
// TODO: check contents of the DBF
fs.unlinkSync(tmpfile);
done();
});
@ -849,7 +965,38 @@ test('KML format, unauthenticated', function(done){
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
// TODO: check for actual content, at least try to uncompress..
var row0 = res.body;
var checkfields = {'Name':1, 'address':1, 'cartodb_id':1, 'the_geom':0, 'the_geom_webmercator':0};
for ( var f in checkfields ) {
if ( checkfields[f] ) {
assert.ok(row0.indexOf('SimpleData name="'+ f + '"') != -1, "result does not include '" + f + "'");
} else {
assert.ok(row0.indexOf('SimpleData name="'+ f + '"') == -1, "result includes '" + f + "'");
}
}
done();
});
});
test('KML format, skipfields', function(done){
assert.response(app, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
var row0 = res.body;
var checkfields = {'Name':1, 'address':0, 'cartodb_id':0, 'the_geom':0, 'the_geom_webmercator':0};
for ( var f in checkfields ) {
if ( checkfields[f] ) {
assert.ok(row0.indexOf('SimpleData name="'+ f + '"') != -1, "result does not include '" + f + "'");
} else {
assert.ok(row0.indexOf('SimpleData name="'+ f + '"') == -1, "result includes '" + f + "'");
}
}
done();
});
});