cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/staging'

Browse Source
This commit is contained in:
Luis Bosque 2013-05-20 14:41:41 +02:00
commit b98a32c296
11 changed files with 132 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
NEWS.md
View File

@ -1,13 +1,23 @@
1.1.10
1.2.1
------
1.2.0
------
* Multilayer API changes
* Layers passed by index in grid fetching url
* Interactivity only specified in layergroup config
* Embed cache_buster within token
* Use ISO format for last_modified timestamp
* Expected LZMA encoding changed to base64
1.1.10
------
* Fix regression with default interactivity parameter (#74)
* More verbose logging for SQL api connection errors
* Write stats for multilayer map token request
1.1.9
-----
* Handle SQL API errors by requesting no Varnish cache
* Fix X-Cache-Channel for multilayer (by token) responses
* Add last_modified field to layergroup creation response (#72)

38
lib/cartodb/server_options.js
View File

@ -105,7 +105,14 @@ module.exports = function(){
me.affectedTables = function (username, api_key, sql, callback) {
var sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// Replace mapnik tokens
sql = sql.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
.replace(RegExp('!pixel_width!', 'g'), '1')
.replace(RegExp('!pixel_height!', 'g'), '1')
;
// Pass to CDB_QueryTables
sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// call sql api
me.sqlQuery(username, api_key, sql, function(err, rows){
@ -263,7 +270,8 @@ module.exports = function(){
// find last updated
me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
if ( err ) { done(err); return; }
response.last_updated = lastUpdated;
response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format
done(null);
});
});
@ -283,26 +291,17 @@ module.exports = function(){
// TODO: check ?
//console.log("type of req.query.lzma is " + typeof(req.query.lzma));
//console.log("req.query.lzma is " + req.query.lzma);
// Decode
var lzma = [];
for (var i=0; i<req.query.lzma.length/2; ++i) {
var hex = req.query.lzma.substring(i*2, i*2+2);
var num = parseInt(hex, "16");
if ( num > 127 ) num = 127-num;
//console.log(i + " hex: " + hex + " decodes as " + num);
lzma.push( num );
}
// Decode (from base64)
var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 })
// Decompress
//console.log("LZMA decompression starts with " + lzma);
LZMA.decompress(
lzma,
function(result) {
//console.log("LZMA decompression completed, payload: "); console.dir(result);
try {
req.query = JSON.parse(result);
delete req.query.lzma
_.extend(req.query, JSON.parse(result))
me.req2params(req, callback);
} catch (err) {
callback(new Error('Error parsing lzma as JSON: ' + err));
@ -316,12 +315,19 @@ module.exports = function(){
}
// Whitelist query parameters and attach format
var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert' ];
var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert', 'config' ];
var bad_query = _.difference(_.keys(req.query), good_query);
_.each(bad_query, function(key){ delete req.query[key]; });
req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
if ( req.params.token ) {
//console.log("Request parameters include token " + req.params.token);
var tksplit = req.params.token.split(':');
req.params.token = tksplit[0];
if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1];
}
// bring all query values onto req.params object
_.extend(req.params, req.query);

11
npm-shrinkwrap.json generated
View File

@ -193,10 +193,10 @@
"version": "1.3.3"
},
"windshaft": {
"version": "0.11.2",
"version": "0.12.0",
"dependencies": {
"grainstore": {
"version": "0.12.0",
"version": "0.13.0",
"dependencies": {
"carto": {
"version": "0.9.3-cdb3",
@ -288,7 +288,7 @@
}
},
"tilelive": {
"version": "4.3.3",
"version": "4.4.2",
"dependencies": {
"optimist": {
"version": "0.3.7",
@ -307,9 +307,6 @@
"version": "0.5.0",
"from": "git://github.com/Vizzuality/tilelive-mapnik.git#6a360ee50",
"dependencies": {
"generic-pool": {
"version": "2.0.3"
},
"eio": {
"version": "0.1.0"
},
@ -327,7 +324,7 @@
"version": "0.0.5"
},
"generic-pool": {
"version": "1.0.12"
"version": "2.0.3"
},
"redis": {
"version": "0.7.2"

6
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.1.10",
"version": "1.2.1",
"description": "A map tile server for CartoDB",
"url": "https://github.com/Vizzuality/Windshaft-cartodb",
"licenses": [{
@ -21,9 +21,9 @@
"cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"windshaft" : "~0.11.2",
"windshaft" : "~0.12.0",
"step": "0.0.x",
"generic-pool": "~1.0.12",
"generic-pool": "~2.0.3",
"redis": "0.7.2",
"hiredis": "~0.1.14",
"request": "2.9.202",

2
run_tests.sh
View File

@ -76,7 +76,7 @@ fi
PATH=node_modules/.bin/:$PATH
echo "Running tests"
mocha -u tdd ${MOCHA_OPTS} ${TESTS}
mocha -t 10000 -u tdd ${MOCHA_OPTS} ${TESTS}
ret=$?
cleanup

59
test/acceptance/multilayer.js
View File

@ -21,6 +21,8 @@ suite('multilayer', function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu
var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
suiteSetup(function(done){
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
@ -34,17 +36,19 @@ suite('multilayer', function() {
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.0.1'
cartocss_version: '2.0.1',
interactivity: 'cartodb_id'
} },
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2',
cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
cartocss_version: '2.0.2'
cartocss_version: '2.0.2',
interactivity: 'cartodb_id'
} }
]
};
var expected_token = "d55208dccb30b5ff972562f563db0d22";
var expected_token = "e34dd7e235138a062f8ba7ad051aa3a7";
Step(
function do_post()
{
@ -65,15 +69,9 @@ suite('multilayer', function() {
+ layergroup.layers[1].options.sql
+ '$windshaft$)'
});
expectedBody.last_updated = JSON.stringify({
'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max '
+ 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+ qTables + '}\')'
});
assert.equal(parsedBody.last_updated, expected_last_updated);
if ( expected_token ) {
//assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
//assert.equal(parsedBody.last_updated, expectedBody.last_updated);
assert.deepEqual(parsedBody, expectedBody);
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
}
else expected_token = parsedBody.layergroupid;
next(null, res);
@ -84,7 +82,7 @@ suite('multilayer', function() {
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token + '/0/0/0.png',
url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png',
method: 'GET',
headers: {host: 'localhost' },
encoding: 'binary'
@ -116,7 +114,7 @@ suite('multilayer', function() {
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer0/0/0/0.grid.json?interactivity=cartodb_id',
+ '/0/0/0/0.grid.json',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
@ -134,7 +132,7 @@ suite('multilayer', function() {
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer1/0/0/0.grid.json?interactivity=cartodb_id',
+ '/1/0/0/0.grid.json',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
@ -175,12 +173,13 @@ suite('multilayer', function() {
sql: 'select 1 as cartodb_id, '
+ 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
cartocss: '#layer { polygon-fill:red; }',
cartocss_version: '2.0.1'
cartocss_version: '2.0.1',
interactivity: 'cartodb_id'
} }
]
};
var expected_token = "20f5710c00e3a1b0b4950de65ef0d875";
var expected_token = "6d8e4ad5458e2d25cf0eef38e38717a6";
Step(
function do_post()
{
@ -200,15 +199,9 @@ suite('multilayer', function() {
+ layergroup.layers[0].options.sql
+ '$windshaft$)'
});
expectedBody.last_updated = JSON.stringify({
'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max '
+ 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+ qTables + '}\')'
});
assert.equal(parsedBody.last_updated, expected_last_updated);
if ( expected_token ) {
//assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
//assert.equal(parsedBody.last_updated, expectedBody.last_updated);
assert.deepEqual(parsedBody, expectedBody);
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
}
else expected_token = parsedBody.layergroupid;
next(null, res);
@ -219,7 +212,7 @@ suite('multilayer', function() {
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token + '/1/0/0.png',
url: '/tiles/layergroup/' + expected_token + ':cb10/1/0/0.png',
method: 'GET',
headers: {host: 'localhost' },
encoding: 'binary'
@ -236,6 +229,9 @@ suite('multilayer', function() {
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql
.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
.replace(RegExp('!pixel_width!', 'g'), '1')
.replace(RegExp('!pixel_height!', 'g'), '1')
+ '$windshaft$)');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
@ -249,7 +245,7 @@ suite('multilayer', function() {
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token + '/4/0/0.png',
url: '/tiles/layergroup/' + expected_token + ':cb11/4/0/0.png',
method: 'GET',
headers: {host: 'localhost' },
encoding: 'binary'
@ -266,6 +262,9 @@ suite('multilayer', function() {
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql
.replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
.replace('!pixel_width!', '1')
.replace('!pixel_height!', '1')
+ '$windshaft$)');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
@ -280,7 +279,7 @@ suite('multilayer', function() {
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer0/1/0/0.grid.json?interactivity=cartodb_id',
+ '/0/1/0/0.grid.json',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
@ -298,7 +297,7 @@ suite('multilayer', function() {
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer0/4/0/0.grid.json?interactivity=cartodb_id',
+ '/0/4/0/0.grid.json',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
@ -334,7 +333,7 @@ suite('multilayer', function() {
version: '1.0.0',
layers: [
{ options: {
sql: 'select 1 as cartodb_id, '
sql: 'select 1 as cartodb_id, !pixel_height! as h'
+ 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
cartocss: '#layer { polygon-fill:red; }',
cartocss_version: '2.0.1'
@ -397,6 +396,8 @@ suite('multilayer', function() {
errors.push(err.message);
console.log("Error: " + err);
}
// trip epoch
expected_token = expected_token.split(':')[0];
redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);

43
test/acceptance/server.js
View File

@ -7,37 +7,15 @@ var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
var http = require('http');
var LZMA = require('lzma/lzma_worker.js').LZMA;
var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js');
require(__dirname + '/../support/test_helper');
var helper = require(__dirname + '/../support/test_helper');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
// Utility function to compress & encode LZMA
function lzma_compress_to_hex(payload, mode, callback) {
var HEX = [ '0','1','2','3','4','5','6','7',
'8','9','a','b','c','d','e','f' ];
LZMA.compress(payload, mode,
function(ints) {
for (var i=0; i<ints.length; ++i) {
if ( ints[i] < 0 ) ints[i] = 127-ints[i];
var hi = ints[i] >> 4;
var lo = ints[i] & 0x0f;
ints[i] = HEX[hi] + HEX[lo];
};
var hex = ints.join('');
callback(null, hex);
},
function(percent) {
//console.log("Compressing: " + percent + "%");
}
);
}
suite('server', function() {
var redis_client = redis.createClient(global.environment.redis.port);
@ -814,27 +792,32 @@ suite('server', function() {
function compressQuery () {
//console.log("Compressing starts");
var next = this;
lzma_compress_to_hex(JSON.stringify(qo), 1, this);
//cosole.log("compress returned " + x );
helper.lzma_compress_to_base64(JSON.stringify(qo), 1, this);
},
function sendRequest(err, lzma) {
if ( err ) throw err;
var next = this;
//console.log("Compressing ends: " + typeof(lzma) + " - " + lzma);
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png?lzma=' + lzma,
url: '/tiles/test_table/15/16046/12354.png?lzma=' + encodeURIComponent(lzma),
method: 'GET',
encoding: 'binary'
},{}, this);
},{}, function(res) { next(null, res); });
},
function checkResponse(res) {
function checkResponse(err, res) {
if ( err ) throw err;
var next = this;
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
function(err, similarity) {
if (err) throw err;
done();
next(err);
});
},
function finish(err) {
done(err);
}
);
});

9
test/support/SQLAPIEmu.js
View File

@ -3,11 +3,20 @@ var url = require('url');
var o = function(port, cb) {
this.queries = [];
var that = this;
this.sqlapi_server = http.createServer(function(req,res) {
var query = url.parse(req.url, true).query;
that.queries.push(query);
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else if ( query.q.match('EPOCH.* as max') ) {
// This is the structure of the known query sent by tiler
var row = {
'max': 1234567890123
};
res.write(JSON.stringify({rows: [ row ]}));
} else {
var qs = JSON.stringify(query);
var row = {

18
test/support/test_helper.js
View File

@ -6,6 +6,7 @@
*/
var _ = require('underscore');
var LZMA = require('lzma/lzma_worker.js').LZMA;
// set environment specific variables
global.settings = require(__dirname + '/../../config/settings');
@ -13,4 +14,21 @@ global.environment = require(__dirname + '/../../config/environments/test');
_.extend(global.settings, global.environment);
// Utility function to compress & encode LZMA
function lzma_compress_to_base64(payload, mode, callback) {
LZMA.compress(payload, mode,
function(ints) {
ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
var base64 = new Buffer(ints, 'binary').toString('base64');
callback(null, base64);
},
function(percent) {
//console.log("Compressing: " + percent + "%");
}
);
}
module.exports = {
lzma_compress_to_base64: lzma_compress_to_base64
}

19
test/unit/cartodb/req2params.test.js
View File

@ -63,4 +63,23 @@ suite('req2params', function() {
});
});
test('it should extend params with decoded lzma', function(done) {
var qo = {
style: 'test',
style_version: '2.1.0',
cache_buster: 5
};
test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
opts.req2params({ query: { non_included: 'toberemoved', api_key: 'test', style: 'override', lzma: data }}, function(err, req) {
var query = req.params
assert.equal(qo.style, query.style)
assert.equal(qo.style_version, query.style_version)
assert.equal(qo.cache_buster, query.cache_buster)
assert.equal('test', query.api_key)
assert.equal(undefined, query.non_included)
done();
});
});
});
});

2
tools/reset_styles
View File

@ -16,7 +16,7 @@ this script.
var path = require('path');
// Reset all styles in the store
var grainstore = require('../node_modules/grainstore/lib/grainstore');
var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
var mapnik = require('mapnik');
var redis = require('redis');