cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/staging'

Browse Source
This commit is contained in:
Luis Bosque 2012-11-14 12:48:07 +01:00
commit 597008d9d4
12 changed files with 290 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
NEWS.md
View File

@ -1,5 +1,24 @@
1.1.0 (30/10/12)
1.1.2 (DD//MM//YY)
-----
* CartoCSS versioning
* Fix use of "style_version" with GET (inline styles)
* Enhance 2.0 -> 2.1 transforms:
* styles with no semicolon
* markers shift due to geometry clipping
1.1.1 (DD//MM//YY)
-----
* Add support for persistent client cache headers
* Fix crash on unknown user (#55)
* Add /version entry point
* CartoCSS versioning
* Include style_version in GET /style response
* Support style_version and style_convert parameters in POST /style request
* Support style_version in GET /:z/:x/:y request
1.1.0 (30/10/12)
=======
* Add /version entry point
* CartoCSS versioning
* Include version in GET /style response

12
README.md
View File

@ -80,8 +80,16 @@ Args:
* sql - plain SQL arguments
* interactivity - specify the column to use in UTFGrid
* cache_buster - if needed you can add a cachebuster to make sure you're
rendering new
* cache_buster - Specify an identifier for the internal tile cache.
Requesting tiles with the same cache_buster value may
result in being served a cached version of the tile
(even when requesting a tile for the first time, as tiles
can be prepared in advance)
* cache_policy - Set to "persist" to have the server send an Cache-Control
header requesting caching devices to keep the response
cached as much as possible. This is best used with a
timestamp value in cache_buster for manual control of
updates.
* geom_type - override the cartodb default
* style - override the default map style with Carto

1
cluster.js
View File

@ -44,6 +44,7 @@ var cluster = new Cluster({
host: global.environment.host,
monPort: global.environment.port+1,
monHost: global.environment.host,
timeout: 600000,
noWorkers: 1
});

1
config/environments/test.js.example
View File

@ -10,7 +10,6 @@ var config = {
user: "publicuser",
host: '127.0.0.1',
port: 5432,
srid: 4326,
extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
simplify: true
}

2
lib/cartodb/carto_data.js
View File

@ -126,7 +126,7 @@ module.exports = function() {
that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
},
function(err, data){
if (err) throw err;
//if (err) throw err;
callback(err, data);
}
);

8
lib/cartodb/cartodb_windshaft.js
View File

@ -26,6 +26,14 @@ var CartodbWindshaft = function(serverOptions) {
// boot
var ws = new Windshaft.Server(serverOptions);
// Override getVersion to include cartodb-specific versions
var wsversion = ws.getVersion;
ws.getVersion = function() {
var version = wsversion();
version.windshaft_cartodb = require('../../package.json').version;
return version;
}
/**
* Helper to allow access to the layer to be used in the maps infowindow popup.
*/

11
lib/cartodb/server_options.js
View File

@ -45,8 +45,13 @@ module.exports = function(){
var ttl = global.environment.varnish.ttl || 86400;
Cache.generateCacheChannel(req, function(channel){
res.header('X-Cache-Channel', channel);
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
var cache_policy = req.query.cache_policy;
if ( cache_policy == 'persist' ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
}
cb(null, channel); // add last-modified too ?
});
}
@ -60,7 +65,7 @@ module.exports = function(){
me.req2params = function(req, callback){
// Whitelist query parameters and attach format
var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'api_key', 'style'];
var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version'];
var bad_query = _.difference(_.keys(req.query), good_query);
_.each(bad_query, function(key){ delete req.query[key]; });

46
npm-shrinkwrap.json generated
View File

@ -32,13 +32,13 @@
"version": "0.8.3"
},
"npm": {
"version": "1.1.62",
"version": "1.1.65",
"dependencies": {
"semver": {
"version": "1.0.14"
"version": "1.1.0"
},
"ini": {
"version": "1.0.4"
"version": "1.0.5"
},
"slide": {
"version": "1.1.3"
@ -50,7 +50,7 @@
"version": "1.1.14"
},
"minimatch": {
"version": "0.2.6"
"version": "0.2.8"
},
"nopt": {
"version": "2.0.0"
@ -93,10 +93,10 @@
"version": "2.0.4"
},
"node-gyp": {
"version": "0.6.11"
"version": "0.7.1"
},
"fstream-npm": {
"version": "0.1.2",
"version": "0.1.3",
"dependencies": {
"fstream-ignore": {
"version": "0.0.5"
@ -119,19 +119,24 @@
"version": "0.1.2"
},
"npm-registry-client": {
"version": "0.2.7"
"version": "0.2.10",
"dependencies": {
"couch-login": {
"version": "0.1.15"
}
}
},
"read-package-json": {
"version": "0.1.5"
"version": "0.1.8"
},
"read-installed": {
"version": "0.0.2"
"version": "0.0.3"
},
"glob": {
"version": "3.1.12"
"version": "3.1.14"
},
"init-package-json": {
"version": "0.0.5",
"version": "0.0.6",
"dependencies": {
"promzard": {
"version": "0.2.0"
@ -147,9 +152,6 @@
"retry": {
"version": "0.6.0"
},
"couch-login": {
"version": "0.1.12"
},
"once": {
"version": "1.1.1"
},
@ -180,10 +182,14 @@
"version": "1.3.3"
},
"grainstore": {
"version": "0.9.1",
"version": "0.9.6",
"dependencies": {
"semver": {
"version": "1.1.0"
},
"carto": {
"version": "0.9.2",
"version": "0.9.3-cdb1",
"from": "git://github.com/CartoDB/carto.git#cdb-0.9",
"dependencies": {
"mapnik-reference": {
"version": "5.0.0"
@ -199,7 +205,7 @@
}
},
"millstone": {
"version": "0.5.10",
"version": "0.5.11",
"dependencies": {
"request": {
"version": "2.11.4",
@ -226,7 +232,7 @@
}
},
"srs": {
"version": "0.2.16"
"version": "0.2.17"
},
"zipfile": {
"version": "0.3.2"
@ -245,7 +251,7 @@
}
},
"windshaft": {
"version": "0.6.2",
"version": "0.7.1",
"dependencies": {
"express": {
"version": "2.5.11",
@ -315,7 +321,7 @@
"version": "2.9.202"
},
"mapnik": {
"version": "0.7.14"
"version": "0.7.16"
},
"mocha": {
"version": "1.2.1",

6
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.1.0",
"version": "1.1.2",
"description": "A map tile server for CartoDB",
"url": "https://github.com/Vizzuality/Windshaft-cartodb",
"licenses": [{
@ -21,8 +21,8 @@
"cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"grainstore" : "~0.9.1",
"windshaft" : "~0.6.2",
"grainstore" : "~0.9.6",
"windshaft" : "~0.7.0",
"step": "0.0.x",
"generic-pool": "1.0.x",
"redis": "0.7.2",

185
test/acceptance/server.js
View File

@ -33,6 +33,35 @@ suite('server', function() {
}, function() { done(); });
});
/////////////////////////////////////////////////////////////////////////////////
//
// GET VERSION
//
/////////////////////////////////////////////////////////////////////////////////
test("get call to server returns 200", function(done){
assert.response(server, {
url: '/version',
method: 'GET'
},{
status: 200
}, function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.hasOwnProperty('windshaft_cartodb'), "No 'windshaft_cartodb' version in " + parsed);
console.log("Windshaft-cartodb: " + parsed.windshaft_cartodb);
assert.ok(parsed.hasOwnProperty('windshaft'), "No 'windshaft' version in " + parsed);
console.log("Windshaft: " + parsed.windshaft);
assert.ok(parsed.hasOwnProperty('grainstore'), "No 'grainstore' version in " + parsed);
console.log("Grainstore: " + parsed.grainstore);
assert.ok(parsed.hasOwnProperty('node_mapnik'), "No 'node_mapnik' version in " + parsed);
console.log("Node-mapnik: " + parsed.node_mapnik);
assert.ok(parsed.hasOwnProperty('mapnik'), "No 'mapnik' version in " + parsed);
console.log("Mapnik: " + parsed.mapnik);
// TODO: check actual versions ?
done();
});
});
/////////////////////////////////////////////////////////////////////////////////
//
// GET STYLE
@ -50,7 +79,7 @@ suite('server', function() {
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, "#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
assert.equal(parsed.version, '2.0.0');
assert.equal(parsed.style_version, '2.0.0');
done();
});
});
@ -72,6 +101,23 @@ suite('server', function() {
});
});
// See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
test("get'ing style of private table should fail on unknown username",
function(done) {
assert.response(server, {
headers: {host: 'unknown_user'},
url: '/tiles/test_table_private_1/style',
method: 'GET'
},{
}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"});
done();
});
});
test("get'ing style of private table should succeed when authenticated",
function(done) {
assert.response(server, {
@ -83,7 +129,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.body);
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, "#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
assert.equal(parsed.version, '2.0.0');
assert.equal(parsed.style_version, '2.0.0');
done();
});
});
@ -201,7 +247,7 @@ suite('server', function() {
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, 'Map {background-color:#fff;}');
//assert.equal(parsed.version, '2.0.0');
assert.equal(parsed.style_version, '2.0.0');
done();
});
@ -215,7 +261,7 @@ suite('server', function() {
url: '/tiles/my_table5/style?map_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: style})
data: querystring.stringify({style: style, style_version: '2.0.2'})
},{
}, function(res) {
@ -230,7 +276,7 @@ suite('server', function() {
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, style);
//assert.equal(parsed.version, '2.0.0');
assert.equal(parsed.style_version, '2.0.2');
done();
});
@ -372,6 +418,23 @@ suite('server', function() {
});
});
// See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
test("get'ing infowindow of private table should fail on unknown username",
function(done) {
assert.response(server, {
headers: {host: 'unknown_user'},
url: '/tiles/test_table_private_1/infowindow',
method: 'GET'
},{
}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"});
done();
});
});
test("get'ing infowindow of private table should succeed when authenticated",
function(done) {
assert.response(server, {
@ -438,6 +501,23 @@ suite('server', function() {
});
});
// See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
test("get'ing grid of private table should fail on unknown username",
function(done) {
assert.response(server, {
headers: {host: 'unknown_user'},
url: '/tiles/test_table_private_1/6/31/24.grid.json',
method: 'GET'
},{
}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"});
done();
});
});
test("get'ing the grid of a private table should succeed when authenticated",
function(done) {
assert.response(server, {
@ -455,7 +535,40 @@ suite('server', function() {
// GET TILE
//
/////////////////////////////////////////////////////////////////////////////////
test("should send Cache-Control header with short expiration by default", function(done){
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.png',
method: 'GET'
},{
status: 200,
}, function(res) {
var cc = res.headers['cache-control'];
assert.ok(cc);
//assert.equal(cc, 'public,max-age=31536000'); // 1 year
assert.ok(cc.match('no-cache'), cc);
assert.ok(cc.match('must-revalidate'), cc);
assert.ok(cc.match('public'), cc);
done();
});
});
test("should send Cache-Control header with long expiration when requested", function(done){
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.png?cache_policy=persist',
method: 'GET'
},{
status: 200,
}, function(res) {
var cc = res.headers['cache-control'];
assert.ok(cc);
assert.equal(cc, 'public,max-age=31536000'); // 1 year
done();
});
});
test("get'ing a tile with default style should return an image", function(done){
assert.response(server, {
headers: {host: 'localhost'},
@ -537,6 +650,25 @@ suite('server', function() {
});
});
test("get'ing a tile with data from private table should fail on unknown username", function(done){
var sql = querystring.stringify({
sql: "SELECT * FROM test_table_private_1",
cache_buster:2 // this is to avoid getting the cached response
});
assert.response(server, {
headers: {host: 'unknown_user'},
url: '/tiles/gadm4/6/31/24.png?' + sql,
method: 'GET'
},{
}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"});
done();
});
});
test("get'ing a tile with data from private table should fail when unauthenticated (uses old redis key)", function(done){
var sql = querystring.stringify({
sql: "SELECT * FROM test_table_private_1",
@ -557,6 +689,47 @@ suite('server', function() {
});
});
var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}";
var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
test("get'ing a tile with url specified 2.0.0 style should return an expected tile", function(done){
var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid
method: 'GET',
encoding: 'binary'
},{}, function(res){
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
function(err, similarity) {
if (err) throw err;
done();
});
});
});
test("get'ing a tile with url specified 2.1.0 style should return an expected tile", function(done){
var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'});
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid
method: 'GET',
encoding: 'binary'
},{}, function(res){
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
function(err, similarity) {
if (err) throw err;
done();
});
});
});
/////////////////////////////////////////////////////////////////////////////////
//
// DELETE CACHE

28
test/support/assert.js
View File

@ -8,29 +8,35 @@ var exec = require('child_process').exec;
var assert = module.exports = exports = require('assert');
assert.imageEqualsFile = function(buffer, file_b, callback) {
//
// @param tol tolerated color distance as a percent over max channel value
// by default this is zero. For meaningful values, see
// http://www.imagemagick.org/script/command-line-options.php#metric
//
assert.imageEqualsFile = function(buffer, file_b, tol, callback) {
if (!callback) callback = function(err) { if (err) throw err; };
file_b = path.resolve(file_b);
var file_a = '/tmp/' + (Math.random() * 1e16);
var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable
var err = fs.writeFileSync(file_a, buffer, 'binary');
if (err) throw err;
exec('compare -metric PSNR "' + file_a + '" "' +
var fuzz = tol + '%';
exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' +
file_b + '" /dev/null', function(err, stdout, stderr) {
if (err) {
fs.unlinkSync(file_a);
callback(err);
} else {
stderr = stderr.trim();
if (stderr === 'inf') {
fs.unlinkSync(file_a);
callback(null);
var similarity = parseFloat(stderr);
if ( similarity > 0 ) {
var err = new Error('Images not equal(' + similarity + '): ' +
file_a + ' ' + file_b);
err.similarity = similarity;
callback(err);
} else {
var similarity = parseFloat(stderr);
var err = new Error('Images not equal(' + similarity + '): ' +
file_a + ' ' + file_b);
err.similarity = similarity;
callback(err);
fs.unlinkSync(file_a);
callback(null);
}
}
});

17
tools/show_style Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
# TODO: port to node, if you really need it
REDIS_PORT=6379 # default port
if test -z "$2"; then
echo "Usage: $0 <username> <tablename>" >&2
exit 1
fi
username="$1"
tabname="$2"
dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"`
redis-cli get "map_style|${dbname}|${tabname}" | sed -e 's/\\n/\n/g' -e 's/\\//g'