cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/staging'

Browse Source
This commit is contained in:
Luis Bosque 2013-04-04 14:22:47 +02:00
commit 9162d2cd43
16 changed files with 751 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -1,4 +1,6 @@
node_modules
node_modules*
config/environments/*.js
.idea
tools/munin/windshaft.conf
logs/
pids/

3
Makefile
View File

@ -12,7 +12,8 @@ check-local: config/environments/test.js
test/unit/cartodb/redis_pool.test.js \
test/unit/cartodb/req2params.test.js \
test/acceptance/cache_validator.js \
test/acceptance/server.js
test/acceptance/server.js \
test/acceptance/multilayer.js
check-submodules:
for sub in windshaft grainstore mapnik; do \

11
NEWS.md
View File

@ -1,3 +1,14 @@
1.1.9
-----
* Handle SQL API errors by requesting no Varnish cache
* Fix X-Cache-Channel for multilayer (by token) responses
* Add last_modified field to layergroup creation response (#72)
* Deprecate signal handler for USR1, add handler for USR2 (#71)
* Fix support for ampersend characters in CartoCSS
* Add support for LZMA compressed GET parameters
* Add support for creating layergroups via GET
1.1.8
-----
* Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param

6
app.js
View File

@ -46,6 +46,12 @@ ws.on('listening', function() {
console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
});
// DEPRECATED, use SIGUSR2
process.on('SIGUSR1', function() {
console.log('WARNING: handling of SIGUSR1 by Windshaft-CartoDB is deprecated, please send SIGUSR2 instead');
ws.dumpCacheStats();
});
process.on('SIGUSR2', function() {
ws.dumpCacheStats();
});

6
config/environments/test.js.example
View File

@ -48,8 +48,10 @@ var config = {
}
,sqlapi: {
protocol: 'http',
host: 'localhost.lan',
port: 8080,
host: '',
// This port will be used by "make check" for testing purposes
// It must be available
port: 1080,
version: 'v1'
}
,varnish: {

70
lib/cartodb/cache_validator.js
View File

@ -1,8 +1,5 @@
var _ = require('underscore'),
Varnish = require('node-varnish'),
request = require('request'),
crypto = require('crypto'),
channelCache = {},
varnish_queue = null;
function init(host, port) {
@ -18,72 +15,7 @@ function invalidate_db(dbname, table) {
}
}
function generateCacheChannel(req, callback){
var cacheChannel = "";
// use key to call sql api with sql request if present, else just return dbname and table name
// base key
var tableNames = req.params.table;
var dbName = req.params.dbname;
var username = req.headers.host.split('.')[0];
// replace tableNames with the results of the explain if present
if (_.isString(req.params.sql) && req.params.sql != ''){
// initialise MD5 key of sql for cache lookups
var sql_md5 = generateMD5(req.params.sql);
var api = global.environment.sqlapi;
var qs = {};
// use cache if present
if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
callback(channelCache[sql_md5]);
} else{
// strip out windshaft/mapnik inserted sql if present
var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
sql = (sql != null) ? sql[1] : req.params.sql;
// build up api string
var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
// add query to querystring
qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// add api_key if present in tile request (means table is private)
if (_.isString(req.params.map_key) && req.params.map_key != ''){
qs.api_key = req.params.map_key;
}
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){
if (!err && response.statusCode == 200) {
tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
} else {
//oops, no SQL API. Just cache using fallback 'table' key
tableNames = 'table';
}
cacheChannel = buildCacheChannel(dbName,tableNames);
channelCache[sql_md5] = cacheChannel; // store for caching
callback(cacheChannel);
});
}
} else {
cacheChannel = buildCacheChannel(dbName,tableNames);
callback(cacheChannel);
}
}
function buildCacheChannel(dbName, tableNames){
return dbName + ':' + tableNames;
}
function generateMD5(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
module.exports = {
init: init,
invalidate_db: invalidate_db,
generateCacheChannel: generateCacheChannel
invalidate_db: invalidate_db
}

4
lib/cartodb/cartodb_windshaft.js
View File

@ -82,10 +82,10 @@ var CartodbWindshaft = function(serverOptions) {
ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
ws.doCORS(res);
Step(
function(){
function flushCache(){
serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
},
function(err, data){
function sendResponse(err, data){
if (err){
ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE');
//res.send(500);

249
lib/cartodb/server_options.js
View File

@ -3,6 +3,9 @@ var _ = require('underscore')
, cartoData = require('./carto_data')
, Cache = require('./cache_validator')
, mapnik = require('mapnik')
, crypto = require('crypto')
, request = require('request')
, LZMA = require('lzma/lzma_worker.js').LZMA
;
module.exports = function(){
@ -44,6 +47,141 @@ module.exports = function(){
+ me.grainstore.mapnik_version + ")");
}
/* This whole block is about generating X-Cache-Channel { */
// TODO: review lifetime of elements of this cache
// NOTE: by-token indices should only be dropped when
// the corresponding layegroup is dropped, because
// we have no SQL after layer creation.
me.channelCache = {};
// Run a query through the SQL api
me.sqlQuery = function (username, api_key, sql, callback) {
var api = global.environment.sqlapi;
// build up api string
var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
var qs = { q: sql }
// add api_key if given
if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
if (err){
callback(err);
return;
}
if (res.statusCode != 200) {
var msg = res.body.error ? res.body.error : res.body;
callback(new Error('unexpected response status (' + res.statusCode + ') for sql query: ' + sql));
return;
}
callback(null, body.rows);
});
};
me.findLastUpdated = function (username, api_key, tableNames, callback) {
var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+ tableNames.join(',') + '}\')';
// call sql api
me.sqlQuery(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not find last updated timestamp: ' + msg));
return;
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var last_updated = 0;
if(rows.length !== 0) {
last_updated = rows[0].max || 0;
}
callback(null, last_updated);
});
};
me.affectedTables = function (username, api_key, sql, callback) {
var sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// call sql api
me.sqlQuery(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames.split(',');
callback(null, tableNames);
});
};
me.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};
me.generateMD5 = function(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
me.generateCacheChannel = function(req, callback){
// use key to call sql api with sql request if present, else
// just return dbname and table name base key
var dbName = req.params.dbname;
var cacheKey = [ dbName ];
if ( req.params.token ) cacheKey.push(req.params.token);
else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
cacheKey = cacheKey.join(':');
if ( me.channelCache.hasOwnProperty(cacheKey) ) {
callback(null, me.channelCache[cacheKey]);
return;
}
if ( req.params.token ) {
if ( ! me.channelCache.hasOwnProperty(cacheKey) ) {
callback(new Error('missing channel cache for token ' + req.params.token));
} else {
callback(null, me.channelCache[cacheKey]);
}
return;
}
if ( ! req.params.sql && ! req.params.token ) {
var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
// not worth caching this
callback(null, cacheChannel);
return;
}
if ( ! req.params.sql ) {
callback(new Error("this request doesn't need an X-Cache-Channel generated"));
return;
}
var dbName = req.params.dbname;
var username = req.headers.host.split('.')[0];
// strip out windshaft/mapnik inserted sql if present
var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
sql = (sql != null) ? sql[1] : req.params.sql;
me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) {
if ( err ) { callback(err); return; }
var cacheChannel = me.buildCacheChannel(dbName,tableNames);
me.channelCache[cacheKey] = cacheChannel; // store for caching
callback(null, cacheChannel);
});
};
// Set the cache chanel info to invalidate the cache on the frontend server
//
// @param req The request object.
@ -57,19 +195,57 @@ module.exports = function(){
// skip non-GET requests, or requests for which there's no response
if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
var res = req.res;
var ttl = global.environment.varnish.ttl || 86400;
Cache.generateCacheChannel(req, function(channel){
res.header('X-Cache-Channel', channel);
var cache_policy = req.query.cache_policy;
if ( cache_policy == 'persist' ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
var cache_policy = req.query.cache_policy;
if ( cache_policy == 'persist' ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
var ttl = global.environment.varnish.ttl || 86400;
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
}
me.generateCacheChannel(req, function(err, channel){
if ( ! err ) {
res.header('X-Cache-Channel', channel);
cb(null, channel);
} else {
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
// TODO: evaluate if we should bubble up the error instead
cb(null, 'ERROR');
}
cb(null, channel); // add last-modified too ?
});
}
};
me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
var token = response.layergroupid;
var sql = [];
_.each(mapconfig.layers, function(lyr) {
sql.push(lyr.options.sql);
});
sql = sql.join(';');
var dbName = req.params.dbname;
var usr = req.headers.host.split('.')[0];
var key = req.params.map_key;
var cacheKey = dbName + ':' + token;
me.affectedTables(usr, key, sql, function(err, tableNames) {
if ( err ) { callback(err); return; }
var cacheChannel = me.buildCacheChannel(dbName,tableNames);
me.channelCache[cacheKey] = cacheChannel; // store for caching
// find last updated
me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
if ( err ) { callback(err); return; }
response.last_updated = lastUpdated;
callback(null);
});
});
};
/* X-Cache-Channel generation } */
/**
* Whitelist input and get database name & default geometry type from
@ -79,6 +255,42 @@ module.exports = function(){
*/
me.req2params = function(req, callback){
if ( req.query.lzma ) {
// TODO: check ?
//console.log("type of req.query.lzma is " + typeof(req.query.lzma));
//console.log("req.query.lzma is " + req.query.lzma);
// Decode
var lzma = [];
for (var i=0; i<req.query.lzma.length/2; ++i) {
var hex = req.query.lzma.substring(i*2, i*2+2);
var num = parseInt(hex, "16");
if ( num > 127 ) num = 127-num;
//console.log(i + " hex: " + hex + " decodes as " + num);
lzma.push( num );
}
// Decompress
//console.log("LZMA decompression starts with " + lzma);
LZMA.decompress(
lzma,
function(result) {
//console.log("LZMA decompression completed, payload: "); console.dir(result);
try {
req.query = JSON.parse(result);
me.req2params(req, callback);
} catch (err) {
callback(new Error('Error parsing lzma as JSON: ' + err));
}
},
function(percent) { // progress
//console.log("LZMA decompression " + percent + "%");
}
);
return;
}
// Whitelist query parameters and attach format
var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert' ];
var bad_query = _.difference(_.keys(req.query), good_query);
@ -90,7 +302,8 @@ module.exports = function(){
_.extend(req.params, req.query);
// for cartodb, ensure interactivity is cartodb_id or user specified
req.params.interactivity = req.params.interactivity || 'cartodb_id';
// Don't: https://github.com/Vizzuality/Windshaft-cartodb/issues/69
//req.params.interactivity = req.params.interactivity || 'cartodb_id';
req.params.processXML = function(req, xml, callback) {
var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user;
@ -127,7 +340,7 @@ module.exports = function(){
if (!_.isNull(data))
_.extend(req.params, {geom_type: data});
that.addCacheChannel(req, function(err, chan) {
that.addCacheChannel(req, function(err) {
callback(err, req);
});
}
@ -181,11 +394,17 @@ module.exports = function(){
var that = this;
Step(
function(){
function getParams(){
// this is mostly to compute req.params.dbname
that.req2params(req, this);
},
function(err, data){
if (err) throw err;
function flushInternalCache(err){
// TODO: implement this, see
// http://github.com/Vizzuality/Windshaft-cartodb/issues/73
return true;
},
function flushVarnishCache(err){
if (err) { callback(err); return; }
if(Cache) {
Cache.invalidate_db(req.params.dbname, req.params.table);
}

160
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.1.7",
"version": "1.1.9",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb02",
@ -13,7 +13,7 @@
"version": "1.9.2",
"dependencies": {
"formidable": {
"version": "1.0.12"
"version": "1.0.13"
}
}
},
@ -32,10 +32,10 @@
"version": "0.8.3"
},
"npm": {
"version": "1.2.12",
"version": "1.2.16",
"dependencies": {
"semver": {
"version": "1.1.3"
"version": "1.1.4"
},
"ini": {
"version": "1.1.0"
@ -50,7 +50,7 @@
"version": "1.2.0"
},
"minimatch": {
"version": "0.2.9",
"version": "0.2.11",
"dependencies": {
"sigmund": {
"version": "1.0.0"
@ -71,7 +71,7 @@
"version": "1.0.5"
},
"tar": {
"version": "0.1.16"
"version": "0.1.17"
},
"fstream": {
"version": "0.1.22"
@ -84,7 +84,7 @@
"from": "git://github.com/isaacs/inherits"
},
"mkdirp": {
"version": "0.3.4"
"version": "0.3.5"
},
"read": {
"version": "1.0.4",
@ -95,13 +95,13 @@
}
},
"lru-cache": {
"version": "2.0.4"
"version": "2.3.0"
},
"node-gyp": {
"version": "0.8.4"
"version": "0.9.5"
},
"fstream-npm": {
"version": "0.1.3",
"version": "0.1.4",
"dependencies": {
"fstream-ignore": {
"version": "0.0.6"
@ -124,7 +124,7 @@
"version": "0.1.2"
},
"npm-registry-client": {
"version": "0.2.17",
"version": "0.2.18",
"dependencies": {
"couch-login": {
"version": "0.1.15"
@ -132,13 +132,13 @@
}
},
"read-package-json": {
"version": "0.2.0"
"version": "0.3.0"
},
"read-installed": {
"version": "0.1.1"
},
"glob": {
"version": "3.1.20"
"version": "3.1.21"
},
"init-package-json": {
"version": "0.0.6",
@ -175,6 +175,9 @@
},
"opener": {
"version": "1.3.0"
},
"chmodr": {
"version": "0.1.0"
}
}
}
@ -186,79 +189,82 @@
"underscore": {
"version": "1.3.3"
},
"grainstore": {
"windshaft": {
"version": "0.11.1",
"dependencies": {
"carto": {
"version": "0.9.3-cdb2",
"from": "git://github.com/CartoDB/carto.git#cdb-0.9",
"grainstore": {
"version": "0.12.0",
"dependencies": {
"mapnik-reference": {
"version": "5.0.0-cdb1",
"from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0"
},
"xml2js": {
"version": "0.1.14",
"carto": {
"version": "0.9.3-cdb3",
"from": "git://github.com/CartoDB/carto.git#cdb-0.9.3-cdb3",
"dependencies": {
"sax": {
"version": "0.5.2"
"mapnik-reference": {
"version": "5.0.0-cdb1",
"from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0"
},
"xml2js": {
"version": "0.1.14",
"dependencies": {
"sax": {
"version": "0.5.2"
}
}
}
}
}
}
},
"mapnik-reference": {
"version": "5.0.4"
},
"millstone": {
"version": "0.5.15",
"dependencies": {
"request": {
"version": "2.11.4",
},
"mapnik-reference": {
"version": "5.0.4"
},
"millstone": {
"version": "0.5.15",
"dependencies": {
"form-data": {
"version": "0.0.3",
"generic-pool": {
"version": "2.0.3"
},
"request": {
"version": "2.12.0",
"dependencies": {
"combined-stream": {
"form-data": {
"version": "0.0.3",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
"combined-stream": {
"version": "0.0.3",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
}
}
},
"async": {
"version": "0.1.9"
}
}
},
"async": {
"version": "0.1.9"
"mime": {
"version": "1.2.7"
}
}
},
"srs": {
"version": "0.2.20"
},
"zipfile": {
"version": "0.3.4"
},
"sqlite3": {
"version": "2.1.7"
},
"mime": {
"version": "1.2.7"
"version": "1.2.9"
},
"mkdirp": {
"version": "0.3.5"
}
}
},
"srs": {
"version": "0.2.20"
},
"zipfile": {
"version": "0.3.4"
},
"sqlite3": {
"version": "2.1.5"
},
"mime": {
"version": "1.2.9"
},
"mkdirp": {
"version": "0.3.5"
}
}
}
}
},
"windshaft": {
"version": "0.9.1",
"dependencies": {
},
"express": {
"version": "2.5.11",
"dependencies": {
@ -266,7 +272,7 @@
"version": "1.9.2",
"dependencies": {
"formidable": {
"version": "1.0.12"
"version": "1.0.13"
}
}
},
@ -298,12 +304,18 @@
}
},
"tilelive-mapnik": {
"version": "0.3.3-cdb2",
"from": "git://github.com/Vizzuality/tilelive-mapnik.git#6061c65a",
"version": "0.5.0",
"from": "git://github.com/Vizzuality/tilelive-mapnik.git#6a360ee50",
"dependencies": {
"generic-pool": {
"version": "2.0.3"
},
"eio": {
"version": "0.1.0"
},
"mime": {
"version": "1.2.9"
},
"sphericalmercator": {
"version": "1.0.2"
}
@ -321,7 +333,12 @@
"version": "0.7.2"
},
"hiredis": {
"version": "0.1.14"
"version": "0.1.15",
"dependencies": {
"bindings": {
"version": "1.1.0"
}
}
},
"request": {
"version": "2.9.202"
@ -329,8 +346,11 @@
"mapnik": {
"version": "0.7.22"
},
"lzma": {
"version": "1.2.2"
},
"semver": {
"version": "1.1.3"
"version": "1.1.4"
},
"mocha": {
"version": "1.2.1",

8
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.1.8",
"version": "1.1.9",
"description": "A map tile server for CartoDB",
"url": "https://github.com/Vizzuality/Windshaft-cartodb",
"licenses": [{
@ -21,14 +21,14 @@
"cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"grainstore" : "~0.11.1",
"windshaft" : "~0.9.1",
"windshaft" : "~0.11.1",
"step": "0.0.x",
"generic-pool": "~1.0.12",
"redis": "0.7.2",
"hiredis": "~0.1.14",
"request": "2.9.202",
"mapnik": "~0.7.14"
"mapnik": "~0.7.14",
"lzma": "~1.2.2"
},
"devDependencies": {
"mocha": "1.2.1",

209
test/acceptance/multilayer.js
View File

@ -6,6 +6,7 @@ var querystring = require('querystring');
var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js');
require(__dirname + '/../support/test_helper');
@ -19,8 +20,10 @@ server.setMaxListeners(0);
suite('multilayer', function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
suiteSetup(function(){
suiteSetup(function(done){
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
});
test("layergroup with 2 layers, each with its style", function(done) {
@ -41,7 +44,7 @@ suite('multilayer', function() {
]
};
var expected_token = "d442ca6d3ece793b9c16c02a1d1ea5f2";
var expected_token = "d55208dccb30b5ff972562f563db0d22";
Step(
function do_post()
{
@ -55,9 +58,23 @@ suite('multilayer', function() {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
var expectedBody = { layergroupid: expected_token };
// TODO: check last modified
//expectedBody.layercount = 2;
if ( expected_token ) assert.deepEqual(parsedBody, expectedBody);
// check last modified
var qTables = JSON.stringify({
'q': 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql + ';'
+ layergroup.layers[1].options.sql
+ '$windshaft$)'
});
expectedBody.last_updated = JSON.stringify({
'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max '
+ 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+ qTables + '}\')'
});
if ( expected_token ) {
//assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
//assert.equal(parsedBody.last_updated, expectedBody.last_updated);
assert.deepEqual(parsedBody, expectedBody);
}
else expected_token = parsedBody.layergroupid;
next(null, res);
});
@ -74,6 +91,19 @@ suite('multilayer', function() {
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
// Check X-Cache-Channel
var cc = res.headers['x-cache-channel'];
assert.ok(cc);
var dbname = 'cartodb_test_user_1_db'
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql + ';'
+ layergroup.layers[1].options.sql
+ '$windshaft$)');
assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2,
function(err, similarity) {
next(err);
@ -135,12 +165,177 @@ suite('multilayer', function() {
);
});
test("layergroup can hold substitution tokens", function(done) {
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select 1 as cartodb_id, '
+ 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
cartocss: '#layer { polygon-fill:red; }',
cartocss_version: '2.0.1'
} }
]
};
var expected_token = "20f5710c00e3a1b0b4950de65ef0d875";
Step(
function do_post()
{
var next = this;
assert.response(server, {
url: '/tiles/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
var expectedBody = { layergroupid: expected_token };
// check last modified
var qTables = JSON.stringify({
'q': 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql
+ '$windshaft$)'
});
expectedBody.last_updated = JSON.stringify({
'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max '
+ 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+ qTables + '}\')'
});
if ( expected_token ) {
//assert.equal(parsedBody.layergroupid, expectedBody.layergroupid);
//assert.equal(parsedBody.last_updated, expectedBody.last_updated);
assert.deepEqual(parsedBody, expectedBody);
}
else expected_token = parsedBody.layergroupid;
next(null, res);
});
},
function do_get_tile1(err)
{
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token + '/1/0/0.png',
method: 'GET',
headers: {host: 'localhost' },
encoding: 'binary'
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
// Check X-Cache-Channel
var cc = res.headers['x-cache-channel'];
assert.ok(cc);
var dbname = 'cartodb_test_user_1_db'
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql
+ '$windshaft$)');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
function(err, similarity) {
next(err);
});
});
},
function do_get_tile4(err)
{
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token + '/4/0/0.png',
method: 'GET',
headers: {host: 'localhost' },
encoding: 'binary'
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
// Check X-Cache-Channel
var cc = res.headers['x-cache-channel'];
assert.ok(cc);
var dbname = 'cartodb_test_user_1_db'
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql
+ '$windshaft$)');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
function(err, similarity) {
next(err);
});
});
},
function do_get_grid1(err)
{
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer0/1/0/0.grid.json?interactivity=cartodb_id',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
function(err, similarity) {
next(err);
});
});
},
function do_get_grid4(err)
{
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer0/4/0/0.grid.json?interactivity=cartodb_id',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
function(err, similarity) {
next(err);
});
});
},
function finish(err) {
var errors = [];
if ( err ) {
errors.push(err.message);
console.log("Error: " + err);
}
redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
});
});
}
);
});
suiteTeardown(function(done) {
// This test will add map_style records, like
// 'map_style|null|publicuser|my_table',
redis_client.keys("map_style|*", function(err, matches) {
_.each(matches, function(k) { redis_client.del(k); });
done();
redis_client.del(matches, function(err) {
sqlapi_server.close(done);
});
});
});

155
test/acceptance/server.js
View File

@ -6,6 +6,9 @@ var querystring = require('querystring');
var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
var http = require('http');
var LZMA = require('lzma/lzma_worker.js').LZMA;
var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js');
require(__dirname + '/../support/test_helper');
@ -14,9 +17,31 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
// Utility function to compress & encode LZMA
function lzma_compress_to_hex(payload, mode, callback) {
var HEX = [ '0','1','2','3','4','5','6','7',
'8','9','a','b','c','d','e','f' ];
LZMA.compress(payload, mode,
function(ints) {
for (var i=0; i<ints.length; ++i) {
if ( ints[i] < 0 ) ints[i] = 127-ints[i];
var hi = ints[i] >> 4;
var lo = ints[i] & 0x0f;
ints[i] = HEX[hi] + HEX[lo];
};
var hex = ints.join('');
callback(null, hex);
},
function(percent) {
//console.log("Compressing: " + percent + "%");
}
);
}
suite('server', function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
var default_style = semver.satisfies(mapnik.versions.mapnik, '<2.1.0')
?
@ -30,7 +55,8 @@ suite('server', function() {
var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}";
var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
suiteSetup(function(){
suiteSetup(function(done){
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
});
/////////////////////////////////////////////////////////////////////////////////
@ -504,11 +530,24 @@ suite('server', function() {
//
/////////////////////////////////////////////////////////////////////////////////
test("get'ing a json with default style should return an grid", function(done){
test("get'ing a grid with no interactivity should fail", function(done){
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.grid.json',
method: 'GET'
},{}, function(res) {
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body), {"error":"Missing interactivity parameter"});
done();
});
});
test("get'ing a json with default style should return an grid", function(done){
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.grid.json?interactivity=cartodb_id',
method: 'GET'
},{
status: 200,
headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8',
@ -519,7 +558,7 @@ suite('server', function() {
test("get'ing a json with default style should return an grid", function(done){
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.grid.json',
url: '/tiles/gadm4/6/31/24.grid.json?interactivity=cartodb_id',
method: 'GET'
},{
status: 200,
@ -528,10 +567,13 @@ suite('server', function() {
});
test("get'ing a json with default style and sql should return a constrained grid", function(done){
var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE codineprov = '08'"})
var q = querystring.stringify({
interactivity: 'cartodb_id',
sql: "SELECT * FROM gadm4 WHERE codineprov = '08'"
})
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.grid.json?' + sql,
url: '/tiles/gadm4/6/31/24.grid.json?' + q,
method: 'GET'
},{
status: 200,
@ -543,7 +585,7 @@ suite('server', function() {
function(done) {
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table_private_1/6/31/24.grid.json',
url: '/tiles/test_table_private_1/6/31/24.grid.json?interactivity=cartodb_id',
method: 'GET'
},{}, function(res) {
// 401 Unauthorized
@ -557,7 +599,7 @@ suite('server', function() {
function(done) {
assert.response(server, {
headers: {host: 'unknown_user'},
url: '/tiles/test_table_private_1/6/31/24.grid.json',
url: '/tiles/test_table_private_1/6/31/24.grid.json?interactivity=cartodb_id',
method: 'GET'
},{
}, function(res) {
@ -573,7 +615,7 @@ suite('server', function() {
function(done) {
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table_private_1/6/31/24.grid.json?map_key=1234',
url: '/tiles/test_table_private_1/6/31/24.grid.json?map_key=1234&interactivity=cartodb_id',
method: 'GET'
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@ -778,6 +820,41 @@ suite('server', function() {
});
});
test("get'ing a tile with url specified 2.1.0 style (lzma version)", function(done){
var qo = {
style: test_style_black_210,
style_version: '2.1.0',
cache_buster: 5
};
Step (
function compressQuery () {
//console.log("Compressing starts");
var next = this;
lzma_compress_to_hex(JSON.stringify(qo), 1, this);
//cosole.log("compress returned " + x );
},
function sendRequest(err, lzma) {
//console.log("Compressing ends: " + typeof(lzma) + " - " + lzma);
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png?lzma=' + lzma,
method: 'GET',
encoding: 'binary'
},{}, this);
},
function checkResponse(res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
function(err, similarity) {
if (err) throw err;
done();
});
}
);
});
// See http://github.com/Vizzuality/Windshaft-cartodb/issues/57
test("GET'ing a tile as anonymous with style set by POST", function(done){
var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'});
@ -916,6 +993,66 @@ suite('server', function() {
);
});
test("uses sqlapi to figure source data of query", function(done){
var qo = {
sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator "
+ "FROM gadm4 g, test_table t "
+ "WHERE g.cartodb_id = t.cartodb_id",
map_key: 1234
};
var sqlapi;
Step(
function sendRequest(err) {
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.png?' + querystring.stringify(qo),
method: 'GET'
},{}, this);
},
function checkResponse(res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
var cc = res.headers['x-cache-channel'];
var dbname = 'cartodb_test_user_1_db'
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.api_key, qo.map_key);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)');
done();
}
);
});
test("requests to skip cache on sqlapi error", function(done){
var qo = {
sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator "
+ ", 'SQLAPIERROR' is not null "
+ "FROM gadm4 g, test_table t "
+ "WHERE g.cartodb_id = t.cartodb_id",
map_key: 1234
};
var sqlapi;
Step(
function sendRequest(err) {
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/gadm4/6/31/24.png?' + querystring.stringify(qo),
method: 'GET'
},{}, this);
},
function checkResponse(res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
// does NOT send an x-cache-channel
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
done();
}
);
});
/////////////////////////////////////////////////////////////////////////////////
//
// DELETE CACHE
@ -1040,7 +1177,7 @@ suite('server', function() {
// 'map_style|null|publicuser|my_table',
redis_client.keys("map_style|*", function(err, matches) {
_.each(matches, function(k) { redis_client.del(k); });
done();
sqlapi_server.close(done);
});
});

29
test/support/SQLAPIEmu.js Normal file
View File

@ -0,0 +1,29 @@
var http = require('http');
var url = require('url');
var o = function(port, cb) {
this.sqlapi_server = http.createServer(function(req,res) {
var query = url.parse(req.url, true).query;
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else {
var qs = JSON.stringify(query);
var row = {
// This is the structure of the known query sent by tiler
'cdb_querytables': '{' + qs + '}',
'max': qs
};
res.write(JSON.stringify({rows: [ row ]}));
}
res.end();
}).listen(port, cb);
};
o.prototype.close = function(cb) {
this.sqlapi_server.close(cb);
};
module.exports = o;

4
test/support/sql/gadm4.sql
View File

@ -97,8 +97,6 @@ ALTER TABLE ONLY gadm4
CREATE INDEX bdll25_provincias_4326_2_the_geom_webmercator_idx ON gadm4 USING gist (the_geom_webmercator);
-- development_cartodb_user_3 role
CREATE USER development_cartodb_user_3;
GRANT ALL ON TABLE gadm4 TO development_cartodb_user_3;
GRANT ALL ON TABLE gadm4 TO test_cartodb_user_1;
GRANT SELECT ON TABLE gadm4 TO publicuser;

18
test/unit/cartodb/req2params.test.js
View File

@ -19,7 +19,7 @@ suite('req2params', function() {
assert.ok(_.isObject(req.query), 'request has query');
assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
assert.ok(req.hasOwnProperty('params'), 'request has params');
assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
assert.ok(!req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
assert.equal(req.params.dbname, 'cartodb_test_user_1_db', 'could forge dbname: '+ req.params.dbname);
assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
done();
@ -33,7 +33,7 @@ suite('req2params', function() {
assert.ok(_.isObject(req.query), 'request has query');
assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
assert.ok(req.hasOwnProperty('params'), 'request has params');
assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
assert.ok(!req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
// database_name for user "localhost" (see test/support/prepare_db.sh)
assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
// unauthenticated request gets no dbuser
@ -49,7 +49,7 @@ suite('req2params', function() {
assert.ok(_.isObject(req.query), 'request has query');
assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
assert.ok(req.hasOwnProperty('params'), 'request has params');
assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
assert.ok(!req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
// database_name for user "localhost" (see test/support/prepare_db.sh)
assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
// id for user "localhost" (see test/support/prepare_db.sh)
@ -62,5 +62,17 @@ suite('req2params', function() {
});
});
});
test('retains interactivity', function(done){
opts.req2params({headers: { host:'localhost' }, query: {interactivity: 'fld'} }, function(err, req) {
if ( err ) { console.log(err); throw new Error(err); }
//console.dir(req);
assert.ok(_.isObject(req.query), 'request has query');
assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
assert.ok(req.hasOwnProperty('params'), 'request has params');
assert.ok(req.params.hasOwnProperty('interactivity'), 'request params has no interactivity');
done();
});
});
});

2
tools/munin/windshaft
View File

@ -67,7 +67,7 @@ for pid in ${pids}; do
log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
if test -e "${log}"; then
kill -USR1 "${pid}"
kill -USR2 "${pid}"
cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l)
if test $cnt -gt $maxcache; then maxcache=$cnt; fi
else