cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix support for long (>64k chars) queries in layergroup creation

Browse Source 
Closes #111. Includes testcase.
This commit is contained in:
Sandro Santilli 2014-01-16 17:20:30 +01:00
parent 09d4467e22
commit 5772c81590
4 changed files with 126 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS.md
View File

@ -3,6 +3,8 @@
Bug fixes:
* Fix support for long (>64k chars) queries in layergroup creation (#111)
1.6.1 -- 2014-01-15
-------------------

10
lib/cartodb/server_options.js
View File

@ -78,7 +78,15 @@ module.exports = function(){
if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
//
// NOTE: using POST to avoid size limits:
// Seehttp://github.com/CartoDB/Windshaft-cartodb/issues/111
//
// TODO: use "host" header to allow IP based specification
// of sqlapi address (and avoid a DNS lookup)
//
request.post({url:sqlapi, body:qs, json:true},
function(err, res, body){
if (err){
console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
callback(err);

65
test/acceptance/multilayer.js
View File

@ -900,6 +900,71 @@ suite('multilayer', function() {
);
});
// SQL strings can be of arbitrary length, when using POST
// See https://github.com/CartoDB/Windshaft-cartodb/issues/111
test("sql string can be very long", function(done){
var long_val = 'pretty';
for (var i=0; i<1024; ++i) long_val += ' long'
long_val += ' string';
var sql = "SELECT ";
for (var i=0; i<16; ++i)
sql += "'" + long_val + "'::text as pretty_long_field_name_" + i + ", ";
sql += "cartodb_id, the_geom_webmercator FROM gadm4 g";
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: sql,
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.0.1'
} }
]
};
var errors = [];
var expected_token;
Step(
function do_post()
{
var data = JSON.stringify(layergroup);
assert.ok(data.length > 1024*64);
var next = this;
assert.response(server, {
url: '/tiles/layergroup?api_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: data
}, {}, function(res) { next(null, res); });
},
function check_result(err, res) {
if ( err ) throw err;
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsedBody = JSON.parse(res.body);
var token_components = parsedBody.layergroupid.split(':');
expected_token = token_components[0];
return null;
},
function cleanup(err) {
if ( err ) errors.push(err.message);
if ( ! expected_token ) return null;
var next = this;
redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
next();
});
});
},
function finish(err) {
if ( err ) errors.push('' + err);
if ( errors.length ) done(new Error(errors.join(',')));
else done(null);
}
);
});
suiteTeardown(function(done) {
// This test will add map_style records, like

71
test/support/SQLAPIEmu.js
View File

@ -5,31 +5,60 @@ var o = function(port, cb) {
this.queries = [];
var that = this;
this.sqlapi_server = http.createServer(function(req,res) {
var query = url.parse(req.url, true).query;
that.queries.push(query);
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else if ( query.q.match('EPOCH.* as max') ) {
// This is the structure of the known query sent by tiler
var row = {
'max': 1234567890.123
};
res.write(JSON.stringify({rows: [ row ]}));
} else {
var qs = JSON.stringify(query);
var row = {
// This is the structure of the known query sent by tiler
'cdb_querytables': '{' + qs + '}',
'max': qs
};
res.write(JSON.stringify({rows: [ row ]}));
}
res.end();
//console.log("server got request with method " + req.method);
var query;
if ( req.method == 'GET' ) {
query = url.parse(req.url, true).query;
that.handleQuery(query, res);
}
else if ( req.method == 'POST') {
var data = '';
req.on('data', function(chunk) {
//console.log("GOT Chunk " + chunk);
data += chunk;
});
req.on('end', function() {
//console.log("Data is: "); console.dir(data);
query = JSON.parse(data);
//console.log("Parsed is: "); console.dir(query);
//console.log("handleQuery is " + that.handleQuery);
that.handleQuery(query, res);
});
}
else {
that.handleQuery('SQLAPIEmu does not support method' + req.method, res);
}
}).listen(port, cb);
};
o.prototype.handleQuery = function(query, res) {
this.queries.push(query);
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else if ( query.q.match('EPOCH.* as max') ) {
// This is the structure of the known query sent by tiler
var row = {
'max': 1234567890.123
};
res.write(JSON.stringify({rows: [ row ]}));
} else {
var qs = JSON.stringify(query);
var row = {
// This is the structure of the known query sent by tiler
'cdb_querytables': '{' + qs + '}',
'max': qs
};
var out_obj = {rows: [ row ]};
var out = JSON.stringify(out_obj);
res.write(out);
}
res.end();
};
o.prototype.close = function(cb) {
this.sqlapi_server.close(cb);
};