cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

migrated to node-varnish, fixed tests, refactor

Browse Source
This commit is contained in:
Simon Tokumine 2011-12-12 18:02:10 +00:00
parent d373124f5c
commit ef15f4b48b
12 changed files with 121 additions and 496 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -8,6 +8,8 @@ Look at lib/cartodb/server_options for more on config
* reads dbname from subdomain and cartodb redis for pretty tile urls
* configures windshaft to publish cartodb_id as the interactivity layer
* gets the default geometry type from the cartodb redis store
* allows tiles to be styled individually
* provides a link to varnish high speed cache
* provides a infowindow endpoint for windshaft
* provides a map_metadata endpoint for windshaft
@ -34,6 +36,7 @@ Args:
* interactivity - specify the column to use in UTFGrid
* cache_buster - if needed you can add a cachebuster to make sure you're rendering new
* geom_type - override the cartodb default
* style - override the default map style with Carto
**STYLE**

8
config/environments/development.js
View File

@ -5,7 +5,7 @@ module.exports.redis = {host: '127.0.0.1',
idleTimeoutMillis: 1,
reapIntervalMillis: 1};
module.exports.windshaft_port = 8181;
module.exports.enable_cors = true;
module.exports.varnish_host = 'localhost';
module.exports.varnish_port = 6082;
module.exports.cache_enabled = true;
module.exports.enable_cors = true;
module.exports.varnish_host = 'localhost';
module.exports.varnish_port = 6082;
module.exports.cache_enabled = false;

2
lib/cartodb/cache_validator.js
View File

@ -1,5 +1,5 @@
var _ = require('underscore'),
Varnish = require('./varnish')
Varnish = require('node-varnish');
var varnish_queue = null;

1
lib/cartodb/carto_data.js
View File

@ -144,7 +144,6 @@ module.exports = function() {
function(err, data) {
if (err) throw err;
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
},
function(err, data){

266
lib/cartodb/varnish.js
View File

@ -1,266 +0,0 @@
/**
* this module implements varnish telnet management protocol
* https://www.varnish-cache.org/trac/wiki/ManagementPort
*/
var net = require('net')
var EventEmitter = require('events').EventEmitter;
function VarnishClient(host, port, ready_callback) {
var self = this;
var ready = false;
var cmd_callback = null;
var client = null;
var connected = false;
var connecting = false;
function log() {
console.log.apply(console, arguments);
}
function connect() {
if(connecting || connected ) return;
connecting = true;
log("VARNISH: connection");
ready = false;
if(!client) {
client = net.createConnection(port, host);
client.on('connect', function () {
log("VARNISH: connected");
connected = true;
self.emit('connect');
connecting = false;
});
} else {
client.connect(port, host);
}
}
self.connect = connect;
connect();
client.on('data', function (data) {
data = data.toString();
lines = data.split('\n', 2);
if(lines.length == 2) {
var tk = lines[0].split(' ')
var code = parseInt(tk[0], 10);
var body_length = parseInt(tk[1], 10);
var body = lines[1];
if(!ready) {
ready = true;
ready_callback && ready_callback();
self.emit('ready');
} else if(cmd_callback) {
var c = cmd_callback
cmd_callback = null;
c(null, code, body);
self.emit('response', code, body)
}
}
});
client.on('error', function(err) {
log("[ERROR] some problem in varnish connection", err);
self.emit('error', err);
});
client.on('close', function(e) {
log("[INFO] closed varnish connection");
self.close();
connected = false;
connecting = false;
});
// sends the command to the server
function _send(cmd, callback) {
cmd_callback = callback;
if(connected) {
client.write(cmd + '\n');
} else {
connect();
}
}
// run command if there is no peding response
// fist param of the callback are the error, null
// if all went ok
this.run_cmd = function(cmd, callback) {
if(!connected) {
connect();
}
if(!cmd_callback) {
_send(cmd, callback);
} else {
callback('response pending');
self.emit('error', {
code: 'RESPONSE_PENDING',
message: 'there is a response pending'
});
}
}
// close the connection
this.close = function() {
client.end();
ready = false;
self.emit('close');
}
}
VarnishClient.prototype = new EventEmitter();
function VarnishPool(opts, ready) {
var resources = [];
var available = [];
for(var i = 0; i < opts.pool_size; ++i) {
var v = new VarnishClient(opts.host, opts.port, function() {
resources.push(v);
available.push(v);
});
}
this.run_cmd = function(cmd, callback) {
var v = available.pop()
if(v) {
v.run_cmd(cmd, function(err, status_code, body) {
callback(err, status_code, body);
available.push(v);
ready();
});
} else {
callback('no clients available');
}
}
this.close = function() {
for(var i = 0; i < resources.length; ++i) {
resources[i].close();
}
}
}
function VarnishQueue(host, port) {
var self = this;
var MAX_QUEUE = 2000;
var queue = [];
var ready = false;
var reconnectTimer = null;
var reconnectTries = 0;
var MAX_RECONNECT_TRIES = 120; // 2 minutes
var client = new VarnishClient(host, port);
function log() {
console.log.apply(console, arguments);
}
// attach a dummy callback to error event to avoid nodejs throws an exception and closes the process
self.on('error', function(e) {
log("error", e);
});
client.on('connect', function() {
clearInterval(reconnectTimer);
reconnectTries = 0;
});
client.on('ready', function() {
ready = true;
log('sending pending');
_send_pending();
});
function reconnect() {
ready = false;
clearInterval(reconnectTimer);
reconnectTimer = setInterval(function() {
client.connect();
++reconnectTries;
if(reconnectTries >= MAX_RECONNECT_TRIES) {
self.emit('error', {
code: 'ABORT_RECONNECT',
message: 'max reconnect tries, abouting'
});
clearInterval(reconnectTimer);
}
}, 1000);
}
client.on('close', reconnect);
client.on('error', reconnect);
function _send_pending(empty_callback) {
if(!ready) return;
var c = queue.pop();
if(!c) return;
client.run_cmd(c, function() {
if(queue.length > 0) {
process.nextTick(_send_pending);
} else {
if(empty_callback) {
empty_callback();
}
self.emit('empty');
}
});
}
this.run_cmd = function(cmd) {
queue.push(cmd);
if(queue.length > MAX_QUEUE) {
console.log("varnish command queue too long, removing commands");
self.emit('error', {code: 'TOO_LONG', message: "varnish command queue too long, removing commands"});
queue.pop();
}
if(ready) {
_send_pending();
}
}
this.end = function() {
_send_pending(function() {
client.close();
});
}
}
VarnishQueue.prototype = new EventEmitter();
/*
var queue = new VarnishQueue('localhost', 6082)
setInterval(function() {
queue.run_cmd('purge obj.http.url == /')
}, 10)
*/
/*
v = new VarnishClient('localhost', 6082, function(err) {
console.log('connected');
v.run_cmd('purge obj.http.url == /', function(err, code, body) {
console.log(code);
console.log(body);
v.close();
});
});
pool = new VarnishPool({
host: 'locahost',
port: 6082,
pool_size: 5
});
/*
v.close();
*/
module.exports = {
VarnishClient: VarnishClient,
VarnishQueue: VarnishQueue
}

1
package.json
View File

@ -15,6 +15,7 @@
"email": "simon@vizzuality.com"
},
"dependencies": {
"node-varnish": "0.1.0",
"underscore" : "1.1.x",
"windshaft" : "0.4.2",
"step": "0.0.x",

27
test/acceptance/server.js
View File

@ -32,8 +32,6 @@ tests["get'ing blank style returns default style"] = function(){
});
};
tests["post'ing no style returns 400 with errors"] = function(){
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
@ -45,8 +43,6 @@ tests["post'ing no style returns 400 with errors"] = function(){
});
};
tests["post'ing bad style returns 400 with error"] = function(){
assert.response(server, {
url: '/tiles/my_table3/style',
@ -54,13 +50,11 @@ tests["post'ing bad style returns 400 with error"] = function(){
headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: '#my_table3{backgxxxxxround-color:#fff;}'})
},{
status: 400,
status: 500,
body: JSON.stringify(['style.mss:1:11 Unrecognized rule: backgxxxxxround-color'])
});
};
tests["post'ing multiple bad styles returns 400 with error array"] = function(){
assert.response(server, {
url: '/tiles/my_table4/style',
@ -68,13 +62,11 @@ tests["post'ing multiple bad styles returns 400 with error array"] = function(){
headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: '#my_table4{backgxxxxxround-color:#fff;foo:bar}'})
},{
status: 400,
status: 500,
body: JSON.stringify([ 'style.mss:1:11 Unrecognized rule: backgxxxxxround-color', 'style.mss:1:38 Unrecognized rule: foo' ])
});
};
tests["post'ing good style returns 200"] = function(){
assert.response(server, {
url: '/tiles/my_table5/style',
@ -86,9 +78,6 @@ tests["post'ing good style returns 200"] = function(){
});
};
tests["post'ing good style returns 200 then getting returns original style"] = function(){
var style = '#my_table5{background-color:#fff;}';
assert.response(server, {
@ -111,7 +100,6 @@ tests["post'ing good style returns 200 then getting returns original style"] = f
});
};
tests["get'ing blank infowindow returns blank"] = function(){
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
@ -135,9 +123,6 @@ tests["get'ing blank infowindow with callback returns blank with callback"] = fu
};
// note requires:
// SELECT 0
// HSET rails:cartodb_user_123_db:my_table infowindow "this, that, the other"
tests["get'ing completed infowindow with callback returns information with callback"] = function(){
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
@ -172,9 +157,8 @@ tests["get'ing a json with default style should return an grid"] = function(){
});
};
tests["get'ing a json with default style and sql should return a constrained grid"] = function(){
var sql = querystring.stringify({sql: "(SELECT * FROM gadm4 WHERE name_2 = 'Murcia') as foo"})
var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE codineprov = '08'"})
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
url: '/tiles/gadm4/6/31/24.grid.json?' + sql,
@ -186,9 +170,8 @@ tests["get'ing a json with default style and sql should return a constrained gri
};
tests["get'ing a tile with default style and sql should return a constrained image"] = function(){
var sql = querystring.stringify({sql: "(SELECT * FROM gadm4 WHERE name_2 = 'Murcia') as foo"});
var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE codineprov = '08'"});
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
url: '/tiles/gadm4/6/31/24.png?' + sql,
@ -201,7 +184,7 @@ tests["get'ing a tile with default style and sql should return a constrained ima
tests["get'ing a tile with default style and complex sql should return a constrained image"] = function(){
var sql = querystring.stringify({sql: "(SELECT * FROM gadm4 WHERE name_2 = 'Murcia' AND id_1 > 950) as foo"})
var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE codineprov = '08' AND codccaa > 60"})
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
url: '/tiles/gadm4/6/31/24.png?' + sql,

142
test/acceptance/varnish.js
View File

@ -1,142 +0,0 @@
var assert = require('assert');
var net = require('net');
require(__dirname + '/../test_helper');
var varnish = require(__dirname + '/../../lib/cartodb/varnish');
var tests = module.exports = {};
function VarnishEmu(on_cmd_recieved, port) {
var self = this;
var welcome_msg = 'hi, im a varnish emu, right?';
self.commands_recieved = [];
var sockets = [];
var server = net.createServer(function (socket) {
var command = '';
socket.write("200 " + welcome_msg.length + "\n");
socket.write(welcome_msg);
socket.on('data', function(data) {
self.commands_recieved.push(data);
server.commands++;
on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
socket.write('200 0\n');
});
sockets.push(socket);
});
server.commands = 0;
server.listen(port || 0, "127.0.0.1");
server.close_connections = function() {
for(var s in sockets) {
sockets[s].end();
}
};
return server;
}
tests['ok'] = function() {
assert.ok(true);
};
tests['should connect'] = function() {
var ok = false;
var server = VarnishEmu();
server.on('listening', function() {
var client = new varnish.VarnishClient('127.0.0.1', server.address().port);
client.on('connect', function() {
ok = true;
});
});
setTimeout(function() { assert.ok(ok);
server.close();
}, 200);
};
tests['should send a command'] = function() {
var ok = false;
var server = VarnishEmu(function() {
ok = true;
});
server.on('listening', function() {
var client = new varnish.VarnishClient('127.0.0.1', server.address().port);
client.on('ready', function() {
client.run_cmd('purge obj.http.X == test', function(){});
});
});
setTimeout(function() { assert.ok(ok); }, 100);
}
tests['should emit close on server disconect'] = function() {
var ok = false;
var server = VarnishEmu();
server.on('listening', function() {
var client = new varnish.VarnishClient('127.0.0.1', server.address().port);
client.on('ready', function() {
client.on('close', function() { ok = true; });
server.close_connections();
server.close();
});
});
setTimeout(function() { assert.ok(ok); }, 300);
}
tests['should emit response on command'] = function() {
var ok = false;
var server = VarnishEmu()
server.on('listening', function() {
var client = new varnish.VarnishClient('127.0.0.1', server.address().port);
client.on('ready', function() {
client.run_cmd('purge obj.http.X == test', function(){});
client.on('response', function(code, body) {
ok = true;
assert.equal(200, code);
});
});
});
setTimeout(function() { assert.ok(ok); }, 100);
}
tests['should emit error when the user tries to send when thereis a pending command'] = function() {
var ok = false;
var server = VarnishEmu()
server.on('listening', function() {
var client = new varnish.VarnishClient('127.0.0.1', server.address().port);
client.on('ready', function() {
client.run_cmd('purge obj.http.X == test', function(){});
client.on('error', function(e) {
ok = true;
assert.equal('RESPONSE_PENDING', e.code);
});
client.run_cmd('purge obj.http.X == test', function(){});
});
});
setTimeout(function() { assert.ok(ok); }, 100);
};
//
// queue
//
tests['should send command'] = function() {
var server = VarnishEmu()
server.on('listening', function() {
var queue = new varnish.VarnishQueue('127.0.0.1', server.address().port);
for(var i = 0; i < 5; ++i) {
queue.run_cmd('purge simon_is == gay');
}
});
setTimeout(function() { assert.equal(5, server.commands); }, 100);
}
tests['should send commands on connect'] = function() {
// first create queue
var queue = new varnish.VarnishQueue('127.0.0.1', 1234)
for(var i = 0; i < 10; ++i) {
queue.run_cmd('purge simon_is == gay');
}
// then server
var server = VarnishEmu(null, 1234)
//wait 2 seconds because the client tries every second the reconnection
setTimeout(function() { assert.equal(10, server.commands); }, 2000);
}

9
test/prepare_db.sh
View File

@ -1,16 +1,17 @@
#!/bin/sh
# this script prepare database and redis instance to run accpetance test
# Note: requires a postgis template called template_postgis
echo "preparing redis..."
echo "HSET rails:users:vizzuality id 1" | redis-cli -n 5
echo "HSET rails:users:vizzuality database_name cartodb_test_user_1_db" | redis-cli -n 5
echo 'HSET rails:cartodb_test_user_1_db:my_table infowindow "this, that, the other"' | redis-cli -n 0
echo "preparing postgres..."
dropdb -Upostgres -hlocalhost cartodb_test_user_1_db
createdb -Upostgres -hlocalhost -Ttemplate_postgis -Opostgres -EUTF8 cartodb_test_user_1_db
psql -Upostgres -hlocalhost cartodb_test_user_1_db < windshaft.test.sql
echo "ok, you can run test now"
psql -Upostgres -hlocalhost cartodb_test_user_1_db < ./sql/windshaft.test.sql
psql -Upostgres -hlocalhost cartodb_test_user_1_db < ./sql/gadm4.sql
echo "Finished preparing data. Run tests with expresso."

101
test/sql/gadm4.sql Normal file
View File

File diff suppressed because one or more lines are too long

7
test/windshaft.test.sql → test/sql/windshaft.test.sql
View File

@ -1,12 +1,7 @@
--
-- Windshaft test database
--
-- To use:
--
-- > dropdb -Upostgres -hlocalhost windshaft_test
-- > createdb -Upostgres -hlocalhost -Ttemplate_postgis -Opostgres -EUTF8 windshaft_test
-- > psql -Upostgres -hlocalhost windshaft_test < windshaft.test.sql
--
-- To use run ../prepare_db.sh
-- NOTE: requires a postgis template called template_postgis
--

50
test/unit/cartodb/ttl.test.js
View File

@ -1,50 +0,0 @@
var assert = require('assert')
, _ = require('underscore')
, TTL = require('../../../lib/cartodb/ttl')
, tests = module.exports = {};
tests['all ok'] = function() {
assert.ok(true, "ok");
}
tests['should timeout'] = function() {
var called = false;
var ttl = TTL(function() {
called = true;
}, 0.1);
ttl.start('test');
setTimeout(function() {
assert.ok(called === true, "called");
}, 200);
}
tests['should remove timeout'] = function() {
var called = false;
var ttl = TTL(function() {
called = true;
}, 0.1);
ttl.start('test');
ttl.remove('test');
setTimeout(function() {
assert.ok(called === false, "removed");
}, 200);
}
tests['should renew timeout time'] = function() {
var called = false;
var ttl = TTL(function() {
called = true;
}, 0.3);
ttl.start('test');
setTimeout(function() {
ttl.start('test');
}, 0.5);
setTimeout(function() {
assert.ok(called === false, "removed");
}, 300);
setTimeout(function() {
assert.ok(called === true, "removed");
}, 600);
}