cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'new_cache'

Browse Source
This commit is contained in:
javi 2011-12-05 15:51:31 +01:00
commit 421a0a6efb
13 changed files with 250 additions and 814 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app.js
View File

@ -30,7 +30,6 @@ _.extend(global.settings, global.environment);
var Windshaft = require('windshaft');
var serverOptions = require('./lib/cartodb/server_options');
var Cache = require('./lib/cartodb/tile_cache');
ws = CartodbWindshaft(serverOptions);
ws.listen(global.environment.windshaft_port);

6
config/environments/development.js
View File

@ -5,7 +5,7 @@ module.exports.redis = {host: '127.0.0.1',
idleTimeoutMillis: 1,
reapIntervalMillis: 1};
module.exports.windshaft_port = 8181;
module.exports.lru_cache = true;
module.exports.lru_cache_size = 10000;
module.exports.enable_cors = true;
module.exports.ttl_timeout = 5;
module.exports.varnish_host = 'localhost';
module.exports.varnish_port = 6082;
module.exports.cache_enabled = true;

5
config/environments/production.js
View File

@ -2,6 +2,7 @@ module.exports.name = 'production';
module.exports.postgres = {user: 'tileuser', host: '127.0.0.1', port: 6432};
module.exports.redis = {host: '127.0.0.1', port: 6379};
module.exports.windshaft_port = 8181;
module.exports.lru_cache = true;
module.exports.lru_cache_size = 10000;
module.exports.ttl_timeout = 600; // 10 minutes
module.exports.varnish_host = 'localhost';
module.exports.varnish_port = 6082
module.exports.cache_enabled = true;

5
config/environments/test.js
View File

@ -5,6 +5,7 @@ module.exports.redis = {host: '127.0.0.1',
idleTimeoutMillis: 1,
reapIntervalMillis: 1};
module.exports.windshaft_port = 8080;
module.exports.lru_cache = true;
module.exports.lru_cache_size = 10000;
module.exports.enable_cors = true;
module.exports.varnish_host = '';
module.exports.varnish_port = null;
module.exports.cache_enabled = false;

85
lib/cartodb/cache_validator.js
View File

@ -1,76 +1,17 @@
//
// this module allows to invalidate cache for a table or check if the cache for a table
// is valid
//
// usage:
//
// Cache.getTimestamp('table_name', function(err, timestamp) {
// });
//
// Cache.setTimestamp('table_name', timestamp, function(err, result) {
// });
//
//
var _ = require('underscore'),
RedisPool = require('./redis_pool'),
Step = require('step')
Varnish = require('./varnish')
var varnish_queue = null;
function Cache(redis_opts) {
var redis_pool = new RedisPool(redis_opts);
var me = {}
var redis_options = {
db: 0
}
var redisCommand = function(fn, callback) {
var redis_client;
Step(
function getRedisClient(){
redis_pool.acquire(redis_options.db, this);
},
function getDataForTable(err, redis) {
if (err) throw err;
redis_client = redis;
fn(redis_client, this);
},
function exit(err, data) {
if (!_.isUndefined(redis_client))
redis_pool.release(redis_options.db, redis_client);
if (callback) {
callback(err, data);
}
}
);
}
// get the timestamp for the table
me.getTimestamp = function(database, table, callback) {
redisCommand(function(redis, step) {
redis.GET("cache:" + database + ":" + table + ":last_updated_at", step);
}, function(err, t) {
if(t != null)
callback(err, parseFloat(t));
else
callback(err, t);
});
}
me.setTimestamp = function(database, table, timestamp, callback) {
timestamp = timestamp || new Date().getTime();
redisCommand(function(redis, step) {
redis.SET("cache:" + database + ":" + table + ":last_updated_at", timestamp, step);
}, callback);
}
return me;
};
module.exports = function(redis_opts) {
return new Cache(redis_opts);
function init(host, port) {
varnish_queue = new Varnish.VarnishQueue(host, port);
}
function invalidate_db(dbname) {
varnish_queue.run_cmd('purge obj.http.X-Cache-Channel == ' + dbname);
}
module.exports = {
init: init,
invalidate_db: invalidate_db
}

40
lib/cartodb/cartodb_windshaft.js
View File

@ -2,26 +2,30 @@
var _ = require('underscore')
, Step = require('step')
, Windshaft = require('windshaft')
, Cache = require('./tile_cache');
, Cache = require('./cache_validator')
var CartodbWindshaft = function(serverOptions) {
// set cache if requested
if(serverOptions.lru_cache) {
var lru_cache = Cache.LRUcache(serverOptions.lru_cache_size || 10000,
serverOptions.redis,
serverOptions.ttl_timeout);
_.extend(serverOptions, {
beforeTileRender: lru_cache.beforeTileRender,
afterTileRender: lru_cache.afterTileRender,
cacheStats: lru_cache.getStats,
afterStateChange: lru_cache.afterStateChange
})
// set the cache chanel info to invalidate the cache on the frontend server
serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
res.header('X-Cache-Channel', req.params.dbname);
callback(null, tile, headers);
};
if(serverOptions.cache_enabled) {
console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
serverOptions.afterStateChange = function(req, data, callback) {
Cache.invalidate_db(req.params.dbname);
callback(null, data);
}
}
// boot
var ws = new Windshaft.Server(serverOptions);
/**
* Helper to allow access to the layer to be used in the maps infowindow popup.
*/
@ -40,6 +44,7 @@ var CartodbWindshaft = function(serverOptions) {
);
});
/**
* Helper to allow access to metadata to be used in embedded maps.
*/
@ -58,17 +63,6 @@ var CartodbWindshaft = function(serverOptions) {
);
});
/**
* tile cache stats
*/
ws.get('/cache', function(req, res){
if(serverOptions.cacheStats) {
res.send(serverOptions.cacheStats(req.query.tile_info, req.query.sort_by));
} else {
res.send("Cache no enabled")
}
});
return ws;
}

252
lib/cartodb/lru.js
View File

@ -1,252 +0,0 @@
/**
* see original file: https://github.com/rsms/js-lru
*
* A doubly linked list-based Least Recently Used (LRU) cache. Will keep most
* recently used items while discarding least recently used items when its limit
* is reached.
*
* Licensed under MIT. Copyright (c) 2010 Rasmus Andersson <http://hunch.se/>
* See README.md for details.
*
* Illustration of the design:
*
* entry entry entry entry
* ______ ______ ______ ______
* | head |.newer => | |.newer => | |.newer => | tail |
* | A | | B | | C | | D |
* |______| <= older.|______| <= older.|______| <= older.|______|
*
* removed <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- added
*/
function LRUCache (limit) {
// Current size of the cache. (Read-only).
this.size = 0;
// Maximum number of items this cache can hold.
this.limit = limit;
this._keymap = {};
}
/**
* Put <value> into the cache associated with <key>. Returns the entry which was
* removed to make room for the new entry. Otherwise undefined is returned
* (i.e. if there was enough room already).
*/
LRUCache.prototype.put = function(key, value) {
var entry = {key:key, value:value};
// Note: No protection agains replacing, and thus orphan entries. By design.
this._keymap[key] = entry;
if (this.tail) {
// link previous tail to the new tail (entry)
this.tail.newer = entry;
entry.older = this.tail;
} else {
// we're first in -- yay
this.head = entry;
}
// add new entry to the end of the linked list -- it's now the freshest entry.
this.tail = entry;
if (this.size === this.limit) {
// we hit the limit -- remove the head
return this.shift();
} else {
// increase the size counter
this.size++;
}
}
/**
* Purge the least recently used (oldest) entry from the cache. Returns the
* removed entry or undefined if the cache was empty.
*
* If you need to perform any form of finalization of purged items, this is a
* good place to do it. Simply override/replace this function:
*
* var c = new LRUCache(123);
* c.shift = function() {
* var entry = LRUCache.prototype.shift.call(this);
* doSomethingWith(entry);
* return entry;
* }
*/
LRUCache.prototype.shift = function() {
// todo: handle special case when limit == 1
var entry = this.head;
if (entry) {
if (this.head.newer) {
this.head = this.head.newer;
this.head.older = undefined;
} else {
this.head = undefined;
}
// Remove last strong reference to <entry> and remove links from the purged
// entry being returned:
entry.newer = entry.older = undefined;
// delete is slow, but we need to do this to avoid uncontrollable growth:
delete this._keymap[entry.key];
}
return entry;
}
/**
* Get and register recent use of <key>. Returns the value associated with <key>
* or undefined if not in cache.
*/
LRUCache.prototype.get = function(key, returnEntry) {
// First, find our cache entry
var entry = this._keymap[key];
if (entry === undefined) return; // Not cached. Sorry.
// As <key> was found in the cache, register it as being requested recently
if (entry === this.tail) {
// Already the most recenlty used entry, so no need to update the list
return entry.value;
}
// HEAD--------------TAIL
// <.older .newer>
// <--- add direction --
// A B C <D> E
if (entry.newer) {
if (entry === this.head)
this.head = entry.newer;
entry.newer.older = entry.older; // C <-- E.
}
if (entry.older)
entry.older.newer = entry.newer; // C. --> E
entry.newer = undefined; // D --x
entry.older = this.tail; // D. --> E
if (this.tail)
this.tail.newer = entry; // E. <-- D
this.tail = entry;
return returnEntry ? entry : entry.value;
}
// ----------------------------------------------------------------------------
// Following code is optional and can be removed without breaking the core
// functionality.
/**
* Check if <key> is in the cache without registering recent use. Feasible if
* you do not want to chage the state of the cache, but only "peek" at it.
* Returns the entry associated with <key> if found, or undefined if not found.
*/
LRUCache.prototype.find = function(key) {
return this._keymap[key];
}
/**
* Update the value of entry with <key>. Returns the old value, or undefined if
* entry was not in the cache.
*/
LRUCache.prototype.set = function(key, value) {
var oldvalue, entry = this.get(key, true);
if (entry) {
oldvalue = entry.value;
entry.value = value;
} else {
oldvalue = this.put(key, value);
if (oldvalue) oldvalue = oldvalue.value;
}
return oldvalue;
}
/**
* Remove entry <key> from cache and return its value. Returns undefined if not
* found.
*/
LRUCache.prototype.remove = function(key) {
var entry = this._keymap[key];
if (!entry) return;
delete this._keymap[entry.key]; // need to do delete unfortunately
if (entry.newer && entry.older) {
// relink the older entry with the newer entry
entry.older.newer = entry.newer;
entry.newer.older = entry.older;
} else if (entry.newer) {
// remove the link to us
entry.newer.older = undefined;
// link the newer entry to head
this.head = entry.newer;
} else if (entry.older) {
// remove the link to us
entry.older.newer = undefined;
// link the newer entry to head
this.tail = entry.older;
} else {
this.head = this.tail = undefined;
}
this.size--;
return entry.value;
}
/** Removes all entries */
LRUCache.prototype.removeAll = function() {
// This should be safe, as we never expose strong refrences to the outside
this.head = this.tail = undefined;
this.size = 0;
this._keymap = {};
}
/**
* Return an array containing all keys of entries stored in the cache object, in
* arbitrary order.
*/
if (typeof Object.keys === 'function') {
LRUCache.prototype.keys = function() { return Object.keys(this._keymap); }
} else {
LRUCache.prototype.keys = function() {
var keys = [];
for (var k in this._keymap) keys.push(k);
return keys;
}
}
/**
* Call `fun` for each entry. Starting with the newest entry if `desc` is a true
* value, otherwise starts with the oldest (head) enrty and moves towards the
* tail.
*
* `fun` is called with 3 arguments in the context `context`:
* `fun.call(context, Object key, Object value, LRUCache self)`
*/
LRUCache.prototype.forEach = function(fun, context, desc) {
if (context === true) { desc = true; context = undefined; }
else if (typeof context !== 'object') context = this;
if (desc) {
var entry = this.tail;
while (entry) {
fun.call(context, entry.key, entry.value, this);
entry = entry.older;
}
} else {
var entry = this.head;
while (entry) {
fun.call(context, entry.key, entry.value, this);
entry = entry.newer;
}
}
}
/** Returns a JSON (array) representation */
LRUCache.prototype.toJSON = function() {
var s = [], entry = this.head;
while (entry) {
s.push({key:entry.key.toJSON(), value:entry.value.toJSON()});
entry = entry.newer;
}
return s;
}
/** Returns a String representation */
LRUCache.prototype.toString = function() {
var s = '', entry = this.head;
while (entry) {
s += String(entry.key)+':'+entry.value;
if (entry = entry.newer)
s += ' < ';
}
return s;
}
// Export ourselves
//if (typeof this === 'object') this.LRUCache = LRUCache;
module.exports = LRUCache;

6
lib/cartodb/server_options.js
View File

@ -8,9 +8,9 @@ module.exports = function(){
grainstore: {datasource: global.environment.postgres},
redis: global.environment.redis,
enable_cors: global.environment.enable_cors,
lru_cache: global.environment.lru_cache,
lru_cache_size: global.environment.lru_cache_size,
ttl_timeout: global.environment.ttl_timeout
varnish_host: global.environment.varnish_host,
varnish_port: global.environment.varnish_port,
cache_enabled: global.environment.cache_enabled
};
/**

178
lib/cartodb/tile_cache.js
View File

@ -1,178 +0,0 @@
// tile cache policies
// it exports two cache types:
// 'nocache' implements a pass-troguth cache
// 'lru' implements a LRU cache
var LRUCache = require('./lru'),
CacheValidator = require('./cache_validator'),
TTL = require('./ttl');
var BLANK_TILE_SIZE = 334; // totally transparent tile size in bytes
module.exports.NoCache = function() {
var me = {}
me.beforeTileRender = function(req, res, callback) {
callback(null);
}
me.afterTileRender = function(req, res, tile, headers, callback) {
callback(null, tile, headers);
}
return me;
}
module.exports.LRUcache = function(max_items, redis_opts, ttl_timeout) {
var cache_validator = CacheValidator(redis_opts);
return GenericCache(new LRUCache(max_items), cache_validator, ttl_timeout || 600);
}
// implements a generic cache for tile
// cache_policy should implement set and get methods and optionally getStats
function GenericCache (cache_policy, cache_validator, ttl_timeout) {
var me = {
cache: cache_policy,
cache_validator: cache_validator,
cache_hits: 0,
cache_misses: 0,
cache_not_modified: 0,
current_items: 0,
expired: 0,
cache_invalidated: 0,
max_items: 0
};
// enable ttl
var ttl = TTL(function(key) {
me.remove(key);
me.expired++;
}, ttl_timeout || 60);
function cache_key(req) {
return req.url;
}
function update_items(n) {
me.current_items = n;
if(n > me.max_items) {
me.max_items = n;
}
}
me.remove = function(key) {
ttl.remove(key);
me.cache.remove(key);
update_items(me.cache.size || 0);
}
me.beforeTileRender = function(req, res, callback) {
req.windshaft_start = new Date().getTime();
var key = cache_key(req);
var tile = me.cache.get(key);
if(tile) {
// validate the cache
me.cache_validator.getTimestamp(req.params.dbname, req.params.table, function(err, t) {
if(t !== null && tile.timestamp < t) {
me.cache_misses++;
me.cache_invalidated++;
callback(null);
} else {
// stats
me.cache_hits++;
var timestamp = new Date().getTime();
var delta = timestamp - req.windshaft_start;
tile.cache_time = delta/1000.0;
tile.hits++;
res.header('X-Cache-hit', 'true');
res.header('Last-Modified', new Date(tile.timestamp*1000).toGMTString());
// check 304
var modified_since = req.header('if-modified-since');
if (modified_since) {
modified_since = Date.parse(modified_since);
if(modified_since && modified_since <= timestamp*1000) {
me.cache_not_modified++;
res.statusCode = 304;
res.end();
return;
}
}
res.send(tile.tile, tile.headers, 200);
}
});
} else {
me.cache_misses++;
callback(null);
}
}
me.afterTileRender = function(req, res, tile, headers, callback) {
var timestamp = new Date().getTime();
var delta = timestamp - req.windshaft_start;
var key = cache_key(req);
me.cache.put(key, {
tile: tile,
headers: headers,
timestamp: timestamp/1000.0,
render_time: delta/1000.0,
hits: 0});
ttl.start(key);
update_items(me.cache.size || 0);
callback(null, tile, headers);
}
me.afterStateChange = function(req, data, callback) {
me.cache_validator.setTimestamp(req.params.dbname, req.params.table, new Date().getTime()/1000.0, function(err, t) {
callback(err, data);
});
}
me.getStats = function(include_tile_info, sort_by) {
var total = me.cache_hits + me.cache_misses;
var mem = 0;
var blank_tile_count = 0;
var tile_info = []
me.cache.forEach(function(key, value) {
if(value.tile.length !== undefined) {
mem += value.tile.length;
if(value.tile.length == BLANK_TILE_SIZE) {
blank_tile_count++;
}
}
if(include_tile_info) {
tile_info.push({
key: key,
length: value.tile.length,
hits: value.hits,
render_time: value.render_time,
cache_time: value.cache_time
});
}
});
sort_by = sort_by || 'hits';
tile_info.sort(function(a, b) {
return b[sort_by] - a[sort_by];
});
return {
cache_hits: me.cache_hits,
cache_misses: me.cache_misses,
cache_invalidated: me.cache_invalidated,
cache_not_modified: me.cache_not_modified,
current_items: me.current_items,
expired: me.expired,
max_items: me.max_items,
memory: mem,
memory_per_item: total ? mem/total: 0,
ratio: total ? me.cache_hits/total: 0,
blank_tile_count: blank_tile_count,
blank_tile_size: blank_tile_count*BLANK_TILE_SIZE,
blank_items_ratio: total? blank_tile_count/total: 0,
tiles: tile_info
};
}
return me;
}

47
lib/cartodb/ttl.js
View File

@ -1,47 +0,0 @@
/*
==========================================================
manages timeouts
* timeout in seconds
* on_delete will be called when time expired with the key as first param
usage:
ttl = TTL(function(key) {
console.log("10 seconds expired on " + key");
}, 10);
==========================================================
*/
function TTL(on_delete, timeout) {
var me = {
timeouts: {},
on_delete: on_delete,
timeout: timeout*1000
};
me.start = function(key) {
var t = me.timeouts[key];
if (t) {
clearTimeout(t);
}
me.timeouts[key] = setTimeout(
(function(k) {
return function() {
me.on_delete(k);
};
})(key),
me.timeout);
}
me.remove = function(key) {
var t = me.timeouts[key];
if (t) {
clearTimeout(t);
delete me.timeouts[key];
}
}
return me;
}
module.exports = TTL;

180
lib/cartodb/varnish.js Normal file
View File

@ -0,0 +1,180 @@
/**
* this module implements varnish telnet management protocol
* https://www.varnish-cache.org/trac/wiki/ManagementPort
*/
var net = require('net')
function VarnishClient(host, port, ready_callback) {
var self = this;
var ready = false;
var cmd_callback = null;
var client = null;
function connect() {
ready = false;
client = net.createConnection(port, host);
}
connect();
client.on('data', function(data) {
data = data.toString();
lines = data.split('\n', 2);
if(lines.length == 2) {
var tk = lines[0].split(' ')
var code = parseInt(tk[0], 10);
var body_length = parseInt(tk[1], 10);
var body = lines[1];
if(!ready) {
ready = true;
ready_callback();
} else if(cmd_callback) {
var c = cmd_callback
cmd_callback = null;
c(null, code, body);
}
}
});
client.on('error', function(err) {
console.log("[ERROR] some problem in varnish connection");
});
client.on('close', function() {
self.close();
});
// sends the command to the server
function _send(cmd, callback) {
cmd_callback = callback;
client.write(cmd + '\n');
}
// run command if there is no peding response
// fist param of the callback are the error, null
// if all went ok
this.run_cmd = function(cmd, callback) {
if(!cmd_callback) {
_send(cmd, callback);
} else {
callback('response pending');
}
}
// close the connection
this.close = function() {
client.end();
ready = false;
}
}
function VarnishPool(opts, ready) {
var resources = [];
var available = [];
for(var i = 0; i < opts.pool_size; ++i) {
var v = new VarnishClient(opts.host, opts.port, function() {
resources.push(v);
available.push(v);
});
}
this.run_cmd = function(cmd, callback) {
var v = available.pop()
if(v) {
v.run_cmd(cmd, function(err, status_code, body) {
callback(err, status_code, body);
available.push(v);
ready();
});
} else {
callback('no clients available');
}
}
this.close = function() {
for(var i = 0; i < resources.length; ++i) {
resources[i].close();
}
}
}
function VarnishQueue(host, port) {
var MAX_QUEUE = 2000;
var queue = [];
var ready = false;
var client = new VarnishClient(host, port, function(err) {
ready = true;
_send_pending();
});
function _send_pending(empty_callback) {
var c = queue.pop();
if(!c) return;
client.run_cmd(c, function() {
if(queue.length > 0) {
process.nextTick(_send_pending);
} else {
if(empty_callback) {
empty_callback();
}
}
});
}
this.run_cmd = function(cmd) {
queue.push(cmd);
if(queue.length > MAX_QUEUE) {
console.log("varnish commando queu too long, removing commands");
queue.pop();
}
if(ready) {
_send_pending();
}
}
this.end = function() {
_send_pending(function() {
client.close();
});
}
}
/*
var queue = new VarnishQueue('localhost', 6082)
setInterval(function() {
queue.run_cmd('purge obj.http.url == /')
}, 10)
*/
/*
v = new VarnishClient('localhost', 6082, function(err) {
console.log('connected');
v.run_cmd('purge obj.http.url == /', function(err, code, body) {
console.log(code);
console.log(body);
v.close();
});
});
pool = new VarnishPool({
host: 'locahost',
port: 6082,
pool_size: 5
});
/*
v.close();
*/
module.exports = {
VarnishClient: VarnishClient,
VarnishQueue: VarnishQueue
}

214
test/acceptance/cache.js
View File

@ -1,214 +0,0 @@
var assert = require('assert')
, tests = module.exports = {}
, _ = require('underscore')
, querystring = require('querystring')
, fs = require('fs')
, th = require(__dirname + '/../test_helper')
, CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft')
, http = require('http')
, Step = require('step')
, CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var cached_server = new CartodbWindshaft(serverOptions);
tests["first time a tile is request should not be cached"] = function() {
assert.response(cached_server, {
url: '/tiles/test_table_3/6/31/24.png?geom_type=polygon',
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
assert.ok(res.header('X-Cache-hit') === undefined);
});
}
/*
tests["second time a tile is request should be cached"] = function() {
var cached_server2 = new CartodbWindshaft(serverOptions);
var url= '/tiles/test_table_2/6/31/24.png';
assert.response(cached_server2, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
assert.response(cached_server2, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
assert.ok(res.header('X-Cache-hit') !== undefined);
});
});
}
*/
tests["LRU tile should be removed"] = function() {
var urls = ['/tiles/test_table_2/6/31/24.png',
'/tiles/test_table_2/6/31/25.png',
'/tiles/test_table_2/6/31/26.png',
'/tiles/test_table_2/6/31/27.png'];
//create another server to not take previos test stats into account
var so = _.clone(serverOptions);
_(so).extend({lru_cache: true, lru_cache_size: 3});
var _cached_server = new CartodbWindshaft(so);
function makeReq(url, callback) {
assert.response(_cached_server, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, callback);
}
Step(
function() {
makeReq(urls[0], this);
},
function() {
makeReq(urls[1], this);
},
function() {
makeReq(urls[2], this);
},
function() {
makeReq(urls[3], this);
}, function() {
assert.response(_cached_server, {
url: urls[0],
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
assert.ok(res.header('X-Cache-hit') === undefined);
var st = so.cacheStats()
assert.eql(st.cache_hits, 0);
assert.eql(st.cache_misses, 5);
assert.eql(st.current_items, 3);
});
}
)
}
tests["cache should be invalidated"] = function() {
var url = '/tiles/test_table_2/6/29/27.png';
var cached_server2 = new CartodbWindshaft(serverOptions);
var cache = CacheValidator(global.environment.redis);
assert.response(cached_server2, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
cache.setTimestamp('cartodb_test_user_1_db', 'test_table_2', (new Date().getTime()/1000.0)+100, function() {
assert.response(cached_server2, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(r) {
assert.ok(r.header('X-Cache-hit') === undefined);
});
});
});
};
tests["Last-Modified header should be sent"] = function() {
var cached_server2 = new CartodbWindshaft(serverOptions);
var url= '/tiles/test_table/6/31/22.png';
assert.response(cached_server2, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
assert.response(cached_server2, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
assert.ok(res.header('X-Cache-hit') !== undefined);
var last_modified = res.header('Last-Modified');
assert.ok(last_modified !== undefined);
assert.response(cached_server2, {
url: url,
headers: {
host: 'vizzuality.localhost.lan',
'if-modified-since': last_modified
},
method: 'GET'
}, {
status: 304
});
});
});
}
tests["TTL should invalidate a tile"] = function() {
var url = '/tiles/test_table_2/6/31/24.png';
//create another server to not take previos test stats into account
var so = _.clone(serverOptions);
_(so).extend({ttl_timeout: 1});
var _cached_server = new CartodbWindshaft(so);
// cache it
assert.response(_cached_server, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
console.log("WAIT A LITTLE BIT PLEASE");
// test before invalidating
setTimeout(function() {
var st = so.cacheStats();
assert.eql(st.expired, 0);
}, 500);
// test after invalidation
setTimeout(function() {
assert.response(_cached_server, {
url: url,
headers: {host: 'vizzuality.localhost.lan'},
method: 'GET'
},{
status: 200
}, function(res) {
assert.ok(res.header('X-Cache-hit') === undefined);
var st = so.cacheStats();
assert.eql(st.expired, 1);
});
}, 2000);
});
}

45
test/acceptance/cache_validator.js
View File

@ -1,24 +1,35 @@
var assert = require('assert');
var net = require('net');
require(__dirname + '/../test_helper');
var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator');
var tests = module.exports = {};
function VarnishEmu(on_cmd_recieved) {
var self = this;
var welcome_msg = 'hi, im a varnish emu, right?';
var assert = require('assert'),
tests = module.exports = {},
th = require(__dirname + '/../test_helper'),
CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
self.commands_recieved = [];
var cache = CacheValidator(global.environment.redis);
tests["should get the timestamp it sets"] = function() {
var d = new Date().getTime();
cache.setTimestamp('mydatabase', 'mytable', d, function() {
cache.getTimestamp('mydatabase', 'mytable', function(err, t) {
assert.eql(t, d);
})
var server = net.createServer(function (socket) {
var command = '';
socket.write("200 " + welcome_msg.length + "\n");
socket.write(welcome_msg);
socket.on('data', function(data) {
self.commands_recieved.push(data);
on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
socket.write('200 0\n');
});
});
server.listen(1337, "127.0.0.1");
}
tests["should get null when timestamp is not set"] = function() {
var d = new Date().getTime();
cache.getTimestamp('mydatabase', 'mytable2', function(err, t) {
assert.ok(t === null);
})
tests['working'] = function() { assert.ok(true); };
tests['should call purge on varnish when invalidate database'] = function() {
var varnish = new VarnishEmu(function(cmds) {
assert.ok(cmds.length == 1);
assert.equal('purge obj.http.X-Cache-Channel == test_cache\n', cmds[0]);
});
CacheValidator.init('localhost', 1337);
CacheValidator.invalidate_db('test_cache');
}