diff --git a/app.js b/app.js
index c8e10ad8..7a56422f 100755
--- a/app.js
+++ b/app.js
@@ -30,7 +30,6 @@ _.extend(global.settings, global.environment);
var Windshaft = require('windshaft');
var serverOptions = require('./lib/cartodb/server_options');
-var Cache = require('./lib/cartodb/tile_cache');
ws = CartodbWindshaft(serverOptions);
ws.listen(global.environment.windshaft_port);
diff --git a/config/environments/development.js b/config/environments/development.js
index e02b1c66..4cf40a8c 100644
--- a/config/environments/development.js
+++ b/config/environments/development.js
@@ -5,7 +5,7 @@ module.exports.redis = {host: '127.0.0.1',
idleTimeoutMillis: 1,
reapIntervalMillis: 1};
module.exports.windshaft_port = 8181;
-module.exports.lru_cache = true;
-module.exports.lru_cache_size = 10000;
module.exports.enable_cors = true;
-module.exports.ttl_timeout = 5;
+module.exports.varnish_host = 'localhost';
+module.exports.varnish_port = 6082;
+module.exports.cache_enabled = true;
diff --git a/config/environments/production.js b/config/environments/production.js
index 62e988df..f8cbe150 100644
--- a/config/environments/production.js
+++ b/config/environments/production.js
@@ -2,6 +2,7 @@ module.exports.name = 'production';
module.exports.postgres = {user: 'tileuser', host: '127.0.0.1', port: 6432};
module.exports.redis = {host: '127.0.0.1', port: 6379};
module.exports.windshaft_port = 8181;
-module.exports.lru_cache = true;
-module.exports.lru_cache_size = 10000;
module.exports.ttl_timeout = 600; // 10 minutes
+module.exports.varnish_host = 'localhost';
+module.exports.varnish_port = 6082
+module.exports.cache_enabled = true;
diff --git a/config/environments/test.js b/config/environments/test.js
index 86b9e524..6d912d0c 100644
--- a/config/environments/test.js
+++ b/config/environments/test.js
@@ -5,6 +5,7 @@ module.exports.redis = {host: '127.0.0.1',
idleTimeoutMillis: 1,
reapIntervalMillis: 1};
module.exports.windshaft_port = 8080;
-module.exports.lru_cache = true;
-module.exports.lru_cache_size = 10000;
module.exports.enable_cors = true;
+module.exports.varnish_host = '';
+module.exports.varnish_port = null;
+module.exports.cache_enabled = false;
diff --git a/lib/cartodb/cache_validator.js b/lib/cartodb/cache_validator.js
index 29d53e78..803f9238 100644
--- a/lib/cartodb/cache_validator.js
+++ b/lib/cartodb/cache_validator.js
@@ -1,76 +1,17 @@
-//
-// this module allows to invalidate cache for a table or check if the cache for a table
-// is valid
-//
-// usage:
-//
-// Cache.getTimestamp('table_name', function(err, timestamp) {
-// });
-//
-// Cache.setTimestamp('table_name', timestamp, function(err, result) {
-// });
-//
-//
-
var _ = require('underscore'),
- RedisPool = require('./redis_pool'),
- Step = require('step')
+ Varnish = require('./varnish')
+var varnish_queue = null;
-function Cache(redis_opts) {
-
- var redis_pool = new RedisPool(redis_opts);
-
- var me = {}
-
- var redis_options = {
- db: 0
- }
-
- var redisCommand = function(fn, callback) {
- var redis_client;
- Step(
- function getRedisClient(){
- redis_pool.acquire(redis_options.db, this);
- },
- function getDataForTable(err, redis) {
- if (err) throw err;
- redis_client = redis;
- fn(redis_client, this);
- },
- function exit(err, data) {
- if (!_.isUndefined(redis_client))
- redis_pool.release(redis_options.db, redis_client);
- if (callback) {
- callback(err, data);
- }
- }
- );
- }
-
- // get the timestamp for the table
- me.getTimestamp = function(database, table, callback) {
- redisCommand(function(redis, step) {
- redis.GET("cache:" + database + ":" + table + ":last_updated_at", step);
- }, function(err, t) {
- if(t != null)
- callback(err, parseFloat(t));
- else
- callback(err, t);
- });
- }
-
- me.setTimestamp = function(database, table, timestamp, callback) {
- timestamp = timestamp || new Date().getTime();
- redisCommand(function(redis, step) {
- redis.SET("cache:" + database + ":" + table + ":last_updated_at", timestamp, step);
- }, callback);
- }
-
- return me;
-};
-
-
-module.exports = function(redis_opts) {
- return new Cache(redis_opts);
+function init(host, port) {
+ varnish_queue = new Varnish.VarnishQueue(host, port);
+}
+
+function invalidate_db(dbname) {
+ varnish_queue.run_cmd('purge obj.http.X-Cache-Channel == ' + dbname);
+}
+
+module.exports = {
+ init: init,
+ invalidate_db: invalidate_db
}
diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js
index 195f00b6..26e84eed 100644
--- a/lib/cartodb/cartodb_windshaft.js
+++ b/lib/cartodb/cartodb_windshaft.js
@@ -2,26 +2,30 @@
var _ = require('underscore')
, Step = require('step')
, Windshaft = require('windshaft')
- , Cache = require('./tile_cache');
+ , Cache = require('./cache_validator')
var CartodbWindshaft = function(serverOptions) {
- // set cache if requested
- if(serverOptions.lru_cache) {
- var lru_cache = Cache.LRUcache(serverOptions.lru_cache_size || 10000,
- serverOptions.redis,
- serverOptions.ttl_timeout);
- _.extend(serverOptions, {
- beforeTileRender: lru_cache.beforeTileRender,
- afterTileRender: lru_cache.afterTileRender,
- cacheStats: lru_cache.getStats,
- afterStateChange: lru_cache.afterStateChange
- })
+ // set the cache chanel info to invalidate the cache on the frontend server
+ serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
+ res.header('X-Cache-Channel', req.params.dbname);
+ callback(null, tile, headers);
+ };
+
+ if(serverOptions.cache_enabled) {
+ console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
+ Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
+ serverOptions.afterStateChange = function(req, data, callback) {
+ Cache.invalidate_db(req.params.dbname);
+ callback(null, data);
+ }
}
+
// boot
var ws = new Windshaft.Server(serverOptions);
+
/**
* Helper to allow access to the layer to be used in the maps infowindow popup.
*/
@@ -40,6 +44,7 @@ var CartodbWindshaft = function(serverOptions) {
);
});
+
/**
* Helper to allow access to metadata to be used in embedded maps.
*/
@@ -58,17 +63,6 @@ var CartodbWindshaft = function(serverOptions) {
);
});
- /**
- * tile cache stats
- */
- ws.get('/cache', function(req, res){
- if(serverOptions.cacheStats) {
- res.send(serverOptions.cacheStats(req.query.tile_info, req.query.sort_by));
- } else {
- res.send("Cache no enabled")
- }
- });
-
return ws;
}
diff --git a/lib/cartodb/lru.js b/lib/cartodb/lru.js
deleted file mode 100644
index adb8a8f5..00000000
--- a/lib/cartodb/lru.js
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * see original file: https://github.com/rsms/js-lru
- *
- * A doubly linked list-based Least Recently Used (LRU) cache. Will keep most
- * recently used items while discarding least recently used items when its limit
- * is reached.
- *
- * Licensed under MIT. Copyright (c) 2010 Rasmus Andersson
- * See README.md for details.
- *
- * Illustration of the design:
- *
- * entry entry entry entry
- * ______ ______ ______ ______
- * | head |.newer => | |.newer => | |.newer => | tail |
- * | A | | B | | C | | D |
- * |______| <= older.|______| <= older.|______| <= older.|______|
- *
- * removed <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- added
- */
-function LRUCache (limit) {
- // Current size of the cache. (Read-only).
- this.size = 0;
- // Maximum number of items this cache can hold.
- this.limit = limit;
- this._keymap = {};
-}
-
-/**
- * Put into the cache associated with . Returns the entry which was
- * removed to make room for the new entry. Otherwise undefined is returned
- * (i.e. if there was enough room already).
- */
-LRUCache.prototype.put = function(key, value) {
- var entry = {key:key, value:value};
- // Note: No protection agains replacing, and thus orphan entries. By design.
- this._keymap[key] = entry;
- if (this.tail) {
- // link previous tail to the new tail (entry)
- this.tail.newer = entry;
- entry.older = this.tail;
- } else {
- // we're first in -- yay
- this.head = entry;
- }
- // add new entry to the end of the linked list -- it's now the freshest entry.
- this.tail = entry;
- if (this.size === this.limit) {
- // we hit the limit -- remove the head
- return this.shift();
- } else {
- // increase the size counter
- this.size++;
- }
-}
-
-/**
- * Purge the least recently used (oldest) entry from the cache. Returns the
- * removed entry or undefined if the cache was empty.
- *
- * If you need to perform any form of finalization of purged items, this is a
- * good place to do it. Simply override/replace this function:
- *
- * var c = new LRUCache(123);
- * c.shift = function() {
- * var entry = LRUCache.prototype.shift.call(this);
- * doSomethingWith(entry);
- * return entry;
- * }
- */
-LRUCache.prototype.shift = function() {
- // todo: handle special case when limit == 1
- var entry = this.head;
- if (entry) {
- if (this.head.newer) {
- this.head = this.head.newer;
- this.head.older = undefined;
- } else {
- this.head = undefined;
- }
- // Remove last strong reference to and remove links from the purged
- // entry being returned:
- entry.newer = entry.older = undefined;
- // delete is slow, but we need to do this to avoid uncontrollable growth:
- delete this._keymap[entry.key];
- }
- return entry;
-}
-
-/**
- * Get and register recent use of . Returns the value associated with
- * or undefined if not in cache.
- */
-LRUCache.prototype.get = function(key, returnEntry) {
- // First, find our cache entry
- var entry = this._keymap[key];
- if (entry === undefined) return; // Not cached. Sorry.
- // As was found in the cache, register it as being requested recently
- if (entry === this.tail) {
- // Already the most recenlty used entry, so no need to update the list
- return entry.value;
- }
- // HEAD--------------TAIL
- // <.older .newer>
- // <--- add direction --
- // A B C E
- if (entry.newer) {
- if (entry === this.head)
- this.head = entry.newer;
- entry.newer.older = entry.older; // C <-- E.
- }
- if (entry.older)
- entry.older.newer = entry.newer; // C. --> E
- entry.newer = undefined; // D --x
- entry.older = this.tail; // D. --> E
- if (this.tail)
- this.tail.newer = entry; // E. <-- D
- this.tail = entry;
- return returnEntry ? entry : entry.value;
-}
-
-// ----------------------------------------------------------------------------
-// Following code is optional and can be removed without breaking the core
-// functionality.
-
-/**
- * Check if is in the cache without registering recent use. Feasible if
- * you do not want to chage the state of the cache, but only "peek" at it.
- * Returns the entry associated with if found, or undefined if not found.
- */
-LRUCache.prototype.find = function(key) {
- return this._keymap[key];
-}
-
-/**
- * Update the value of entry with . Returns the old value, or undefined if
- * entry was not in the cache.
- */
-LRUCache.prototype.set = function(key, value) {
- var oldvalue, entry = this.get(key, true);
- if (entry) {
- oldvalue = entry.value;
- entry.value = value;
- } else {
- oldvalue = this.put(key, value);
- if (oldvalue) oldvalue = oldvalue.value;
- }
- return oldvalue;
-}
-
-/**
- * Remove entry from cache and return its value. Returns undefined if not
- * found.
- */
-LRUCache.prototype.remove = function(key) {
- var entry = this._keymap[key];
- if (!entry) return;
- delete this._keymap[entry.key]; // need to do delete unfortunately
- if (entry.newer && entry.older) {
- // relink the older entry with the newer entry
- entry.older.newer = entry.newer;
- entry.newer.older = entry.older;
- } else if (entry.newer) {
- // remove the link to us
- entry.newer.older = undefined;
- // link the newer entry to head
- this.head = entry.newer;
- } else if (entry.older) {
- // remove the link to us
- entry.older.newer = undefined;
- // link the newer entry to head
- this.tail = entry.older;
- } else {
- this.head = this.tail = undefined;
- }
- this.size--;
- return entry.value;
-}
-
-/** Removes all entries */
-LRUCache.prototype.removeAll = function() {
- // This should be safe, as we never expose strong refrences to the outside
- this.head = this.tail = undefined;
- this.size = 0;
- this._keymap = {};
-}
-
-/**
- * Return an array containing all keys of entries stored in the cache object, in
- * arbitrary order.
- */
-if (typeof Object.keys === 'function') {
- LRUCache.prototype.keys = function() { return Object.keys(this._keymap); }
-} else {
- LRUCache.prototype.keys = function() {
- var keys = [];
- for (var k in this._keymap) keys.push(k);
- return keys;
- }
-}
-
-/**
- * Call `fun` for each entry. Starting with the newest entry if `desc` is a true
- * value, otherwise starts with the oldest (head) enrty and moves towards the
- * tail.
- *
- * `fun` is called with 3 arguments in the context `context`:
- * `fun.call(context, Object key, Object value, LRUCache self)`
- */
-LRUCache.prototype.forEach = function(fun, context, desc) {
- if (context === true) { desc = true; context = undefined; }
- else if (typeof context !== 'object') context = this;
- if (desc) {
- var entry = this.tail;
- while (entry) {
- fun.call(context, entry.key, entry.value, this);
- entry = entry.older;
- }
- } else {
- var entry = this.head;
- while (entry) {
- fun.call(context, entry.key, entry.value, this);
- entry = entry.newer;
- }
- }
-}
-
-/** Returns a JSON (array) representation */
-LRUCache.prototype.toJSON = function() {
- var s = [], entry = this.head;
- while (entry) {
- s.push({key:entry.key.toJSON(), value:entry.value.toJSON()});
- entry = entry.newer;
- }
- return s;
-}
-
-/** Returns a String representation */
-LRUCache.prototype.toString = function() {
- var s = '', entry = this.head;
- while (entry) {
- s += String(entry.key)+':'+entry.value;
- if (entry = entry.newer)
- s += ' < ';
- }
- return s;
-}
-
-// Export ourselves
-//if (typeof this === 'object') this.LRUCache = LRUCache;
-
-module.exports = LRUCache;
diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js
index 96e37ee9..e37c2c5e 100644
--- a/lib/cartodb/server_options.js
+++ b/lib/cartodb/server_options.js
@@ -8,9 +8,9 @@ module.exports = function(){
grainstore: {datasource: global.environment.postgres},
redis: global.environment.redis,
enable_cors: global.environment.enable_cors,
- lru_cache: global.environment.lru_cache,
- lru_cache_size: global.environment.lru_cache_size,
- ttl_timeout: global.environment.ttl_timeout
+ varnish_host: global.environment.varnish_host,
+ varnish_port: global.environment.varnish_port,
+ cache_enabled: global.environment.cache_enabled
};
/**
diff --git a/lib/cartodb/tile_cache.js b/lib/cartodb/tile_cache.js
deleted file mode 100644
index 92b1737e..00000000
--- a/lib/cartodb/tile_cache.js
+++ /dev/null
@@ -1,178 +0,0 @@
-// tile cache policies
-// it exports two cache types:
-// 'nocache' implements a pass-troguth cache
-// 'lru' implements a LRU cache
-
-var LRUCache = require('./lru'),
- CacheValidator = require('./cache_validator'),
- TTL = require('./ttl');
-
-var BLANK_TILE_SIZE = 334; // totally transparent tile size in bytes
-
-module.exports.NoCache = function() {
-
- var me = {}
-
- me.beforeTileRender = function(req, res, callback) {
- callback(null);
- }
-
- me.afterTileRender = function(req, res, tile, headers, callback) {
- callback(null, tile, headers);
- }
-
- return me;
-
-}
-
-module.exports.LRUcache = function(max_items, redis_opts, ttl_timeout) {
- var cache_validator = CacheValidator(redis_opts);
- return GenericCache(new LRUCache(max_items), cache_validator, ttl_timeout || 600);
-}
-
-// implements a generic cache for tile
-// cache_policy should implement set and get methods and optionally getStats
-function GenericCache (cache_policy, cache_validator, ttl_timeout) {
-
- var me = {
- cache: cache_policy,
- cache_validator: cache_validator,
- cache_hits: 0,
- cache_misses: 0,
- cache_not_modified: 0,
- current_items: 0,
- expired: 0,
- cache_invalidated: 0,
- max_items: 0
- };
-
- // enable ttl
- var ttl = TTL(function(key) {
- me.remove(key);
- me.expired++;
- }, ttl_timeout || 60);
-
- function cache_key(req) {
- return req.url;
- }
-
- function update_items(n) {
- me.current_items = n;
- if(n > me.max_items) {
- me.max_items = n;
- }
- }
-
- me.remove = function(key) {
- ttl.remove(key);
- me.cache.remove(key);
- update_items(me.cache.size || 0);
- }
-
- me.beforeTileRender = function(req, res, callback) {
- req.windshaft_start = new Date().getTime();
- var key = cache_key(req);
- var tile = me.cache.get(key);
- if(tile) {
- // validate the cache
- me.cache_validator.getTimestamp(req.params.dbname, req.params.table, function(err, t) {
- if(t !== null && tile.timestamp < t) {
- me.cache_misses++;
- me.cache_invalidated++;
- callback(null);
- } else {
- // stats
- me.cache_hits++;
- var timestamp = new Date().getTime();
- var delta = timestamp - req.windshaft_start;
- tile.cache_time = delta/1000.0;
- tile.hits++;
- res.header('X-Cache-hit', 'true');
- res.header('Last-Modified', new Date(tile.timestamp*1000).toGMTString());
- // check 304
- var modified_since = req.header('if-modified-since');
- if (modified_since) {
- modified_since = Date.parse(modified_since);
- if(modified_since && modified_since <= timestamp*1000) {
- me.cache_not_modified++;
- res.statusCode = 304;
- res.end();
- return;
- }
- }
- res.send(tile.tile, tile.headers, 200);
- }
- });
- } else {
- me.cache_misses++;
- callback(null);
- }
- }
-
- me.afterTileRender = function(req, res, tile, headers, callback) {
- var timestamp = new Date().getTime();
- var delta = timestamp - req.windshaft_start;
- var key = cache_key(req);
- me.cache.put(key, {
- tile: tile,
- headers: headers,
- timestamp: timestamp/1000.0,
- render_time: delta/1000.0,
- hits: 0});
- ttl.start(key);
- update_items(me.cache.size || 0);
- callback(null, tile, headers);
- }
-
- me.afterStateChange = function(req, data, callback) {
- me.cache_validator.setTimestamp(req.params.dbname, req.params.table, new Date().getTime()/1000.0, function(err, t) {
- callback(err, data);
- });
- }
-
- me.getStats = function(include_tile_info, sort_by) {
- var total = me.cache_hits + me.cache_misses;
- var mem = 0;
- var blank_tile_count = 0;
- var tile_info = []
- me.cache.forEach(function(key, value) {
- if(value.tile.length !== undefined) {
- mem += value.tile.length;
- if(value.tile.length == BLANK_TILE_SIZE) {
- blank_tile_count++;
- }
- }
- if(include_tile_info) {
- tile_info.push({
- key: key,
- length: value.tile.length,
- hits: value.hits,
- render_time: value.render_time,
- cache_time: value.cache_time
- });
- }
- });
- sort_by = sort_by || 'hits';
- tile_info.sort(function(a, b) {
- return b[sort_by] - a[sort_by];
- });
- return {
- cache_hits: me.cache_hits,
- cache_misses: me.cache_misses,
- cache_invalidated: me.cache_invalidated,
- cache_not_modified: me.cache_not_modified,
- current_items: me.current_items,
- expired: me.expired,
- max_items: me.max_items,
- memory: mem,
- memory_per_item: total ? mem/total: 0,
- ratio: total ? me.cache_hits/total: 0,
- blank_tile_count: blank_tile_count,
- blank_tile_size: blank_tile_count*BLANK_TILE_SIZE,
- blank_items_ratio: total? blank_tile_count/total: 0,
- tiles: tile_info
- };
- }
-
- return me;
-}
diff --git a/lib/cartodb/ttl.js b/lib/cartodb/ttl.js
deleted file mode 100644
index 443911ba..00000000
--- a/lib/cartodb/ttl.js
+++ /dev/null
@@ -1,47 +0,0 @@
-
-/*
- ==========================================================
- manages timeouts
- * timeout in seconds
- * on_delete will be called when time expired with the key as first param
-
- usage:
- ttl = TTL(function(key) {
- console.log("10 seconds expired on " + key");
- }, 10);
- ==========================================================
-*/
-function TTL(on_delete, timeout) {
-
- var me = {
- timeouts: {},
- on_delete: on_delete,
- timeout: timeout*1000
- };
-
- me.start = function(key) {
- var t = me.timeouts[key];
- if (t) {
- clearTimeout(t);
- }
- me.timeouts[key] = setTimeout(
- (function(k) {
- return function() {
- me.on_delete(k);
- };
- })(key),
- me.timeout);
- }
-
- me.remove = function(key) {
- var t = me.timeouts[key];
- if (t) {
- clearTimeout(t);
- delete me.timeouts[key];
- }
- }
-
- return me;
-}
-
-module.exports = TTL;
diff --git a/lib/cartodb/varnish.js b/lib/cartodb/varnish.js
new file mode 100644
index 00000000..977f5bdc
--- /dev/null
+++ b/lib/cartodb/varnish.js
@@ -0,0 +1,180 @@
+
+/**
+ * this module implements varnish telnet management protocol
+ * https://www.varnish-cache.org/trac/wiki/ManagementPort
+ */
+
+var net = require('net')
+
+function VarnishClient(host, port, ready_callback) {
+
+ var self = this;
+ var ready = false;
+ var cmd_callback = null;
+ var client = null;
+
+ function connect() {
+ ready = false;
+ client = net.createConnection(port, host);
+ }
+
+ connect();
+
+ client.on('data', function(data) {
+ data = data.toString();
+ lines = data.split('\n', 2);
+ if(lines.length == 2) {
+ var tk = lines[0].split(' ')
+ var code = parseInt(tk[0], 10);
+ var body_length = parseInt(tk[1], 10);
+ var body = lines[1];
+ if(!ready) {
+ ready = true;
+ ready_callback();
+ } else if(cmd_callback) {
+ var c = cmd_callback
+ cmd_callback = null;
+ c(null, code, body);
+ }
+ }
+
+ });
+
+ client.on('error', function(err) {
+ console.log("[ERROR] some problem in varnish connection");
+ });
+
+ client.on('close', function() {
+ self.close();
+ });
+
+ // sends the command to the server
+ function _send(cmd, callback) {
+ cmd_callback = callback;
+ client.write(cmd + '\n');
+ }
+
+ // run command if there is no peding response
+ // fist param of the callback are the error, null
+ // if all went ok
+ this.run_cmd = function(cmd, callback) {
+ if(!cmd_callback) {
+ _send(cmd, callback);
+ } else {
+ callback('response pending');
+ }
+ }
+
+ // close the connection
+ this.close = function() {
+ client.end();
+ ready = false;
+ }
+
+}
+
+
+function VarnishPool(opts, ready) {
+ var resources = [];
+ var available = [];
+
+ for(var i = 0; i < opts.pool_size; ++i) {
+ var v = new VarnishClient(opts.host, opts.port, function() {
+ resources.push(v);
+ available.push(v);
+ });
+ }
+
+ this.run_cmd = function(cmd, callback) {
+ var v = available.pop()
+ if(v) {
+ v.run_cmd(cmd, function(err, status_code, body) {
+ callback(err, status_code, body);
+ available.push(v);
+ ready();
+ });
+ } else {
+ callback('no clients available');
+ }
+ }
+
+ this.close = function() {
+ for(var i = 0; i < resources.length; ++i) {
+ resources[i].close();
+ }
+ }
+}
+
+function VarnishQueue(host, port) {
+
+ var MAX_QUEUE = 2000;
+ var queue = [];
+ var ready = false;
+
+ var client = new VarnishClient(host, port, function(err) {
+ ready = true;
+ _send_pending();
+ });
+
+ function _send_pending(empty_callback) {
+ var c = queue.pop();
+ if(!c) return;
+ client.run_cmd(c, function() {
+ if(queue.length > 0) {
+ process.nextTick(_send_pending);
+ } else {
+ if(empty_callback) {
+ empty_callback();
+ }
+ }
+ });
+ }
+
+ this.run_cmd = function(cmd) {
+ queue.push(cmd);
+ if(queue.length > MAX_QUEUE) {
+ console.log("varnish commando queu too long, removing commands");
+ queue.pop();
+ }
+ if(ready) {
+ _send_pending();
+ }
+ }
+
+ this.end = function() {
+ _send_pending(function() {
+ client.close();
+ });
+ }
+
+}
+
+/*
+var queue = new VarnishQueue('localhost', 6082)
+setInterval(function() {
+ queue.run_cmd('purge obj.http.url == /')
+}, 10)
+*/
+/*
+v = new VarnishClient('localhost', 6082, function(err) {
+ console.log('connected');
+ v.run_cmd('purge obj.http.url == /', function(err, code, body) {
+ console.log(code);
+ console.log(body);
+ v.close();
+ });
+});
+
+pool = new VarnishPool({
+ host: 'locahost',
+ port: 6082,
+ pool_size: 5
+});
+/*
+v.close();
+*/
+
+module.exports = {
+ VarnishClient: VarnishClient,
+ VarnishQueue: VarnishQueue
+}
diff --git a/test/acceptance/cache.js b/test/acceptance/cache.js
deleted file mode 100644
index c37f7557..00000000
--- a/test/acceptance/cache.js
+++ /dev/null
@@ -1,214 +0,0 @@
-
-var assert = require('assert')
- , tests = module.exports = {}
- , _ = require('underscore')
- , querystring = require('querystring')
- , fs = require('fs')
- , th = require(__dirname + '/../test_helper')
- , CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft')
- , http = require('http')
- , Step = require('step')
- , CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
-
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
-
-var cached_server = new CartodbWindshaft(serverOptions);
-
-tests["first time a tile is request should not be cached"] = function() {
- assert.response(cached_server, {
- url: '/tiles/test_table_3/6/31/24.png?geom_type=polygon',
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
-
- }, function(res) {
- assert.ok(res.header('X-Cache-hit') === undefined);
-
- });
-
-}
-
-/*
-tests["second time a tile is request should be cached"] = function() {
-
- var cached_server2 = new CartodbWindshaft(serverOptions);
- var url= '/tiles/test_table_2/6/31/24.png';
- assert.response(cached_server2, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(res) {
- assert.response(cached_server2, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(res) {
- assert.ok(res.header('X-Cache-hit') !== undefined);
- });
- });
-}
-*/
-
-
-tests["LRU tile should be removed"] = function() {
-
- var urls = ['/tiles/test_table_2/6/31/24.png',
- '/tiles/test_table_2/6/31/25.png',
- '/tiles/test_table_2/6/31/26.png',
- '/tiles/test_table_2/6/31/27.png'];
-
- //create another server to not take previos test stats into account
- var so = _.clone(serverOptions);
- _(so).extend({lru_cache: true, lru_cache_size: 3});
-
- var _cached_server = new CartodbWindshaft(so);
-
- function makeReq(url, callback) {
- assert.response(_cached_server, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, callback);
- }
-
- Step(
- function() {
- makeReq(urls[0], this);
- },
- function() {
- makeReq(urls[1], this);
- },
- function() {
- makeReq(urls[2], this);
- },
- function() {
- makeReq(urls[3], this);
- }, function() {
- assert.response(_cached_server, {
- url: urls[0],
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
-
- }, function(res) {
- assert.ok(res.header('X-Cache-hit') === undefined);
- var st = so.cacheStats()
- assert.eql(st.cache_hits, 0);
- assert.eql(st.cache_misses, 5);
- assert.eql(st.current_items, 3);
- });
- }
- )
-
-}
-
-tests["cache should be invalidated"] = function() {
-
- var url = '/tiles/test_table_2/6/29/27.png';
- var cached_server2 = new CartodbWindshaft(serverOptions);
- var cache = CacheValidator(global.environment.redis);
- assert.response(cached_server2, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(res) {
- cache.setTimestamp('cartodb_test_user_1_db', 'test_table_2', (new Date().getTime()/1000.0)+100, function() {
- assert.response(cached_server2, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(r) {
- assert.ok(r.header('X-Cache-hit') === undefined);
- });
- });
- });
-
-};
-
-tests["Last-Modified header should be sent"] = function() {
- var cached_server2 = new CartodbWindshaft(serverOptions);
- var url= '/tiles/test_table/6/31/22.png';
- assert.response(cached_server2, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(res) {
- assert.response(cached_server2, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(res) {
- assert.ok(res.header('X-Cache-hit') !== undefined);
- var last_modified = res.header('Last-Modified');
- assert.ok(last_modified !== undefined);
- assert.response(cached_server2, {
- url: url,
- headers: {
- host: 'vizzuality.localhost.lan',
- 'if-modified-since': last_modified
- },
- method: 'GET'
- }, {
- status: 304
- });
- });
- });
-}
-
-tests["TTL should invalidate a tile"] = function() {
-
- var url = '/tiles/test_table_2/6/31/24.png';
-
- //create another server to not take previos test stats into account
- var so = _.clone(serverOptions);
- _(so).extend({ttl_timeout: 1});
-
- var _cached_server = new CartodbWindshaft(so);
- // cache it
- assert.response(_cached_server, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(res) {
- console.log("WAIT A LITTLE BIT PLEASE");
-
- // test before invalidating
- setTimeout(function() {
- var st = so.cacheStats();
- assert.eql(st.expired, 0);
- }, 500);
-
- // test after invalidation
- setTimeout(function() {
- assert.response(_cached_server, {
- url: url,
- headers: {host: 'vizzuality.localhost.lan'},
- method: 'GET'
- },{
- status: 200
- }, function(res) {
- assert.ok(res.header('X-Cache-hit') === undefined);
- var st = so.cacheStats();
- assert.eql(st.expired, 1);
- });
- }, 2000);
- });
-}
diff --git a/test/acceptance/cache_validator.js b/test/acceptance/cache_validator.js
index 55bd714e..6e39dd8a 100644
--- a/test/acceptance/cache_validator.js
+++ b/test/acceptance/cache_validator.js
@@ -1,24 +1,35 @@
+var assert = require('assert');
+var net = require('net');
+require(__dirname + '/../test_helper');
+var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator');
+var tests = module.exports = {};
+function VarnishEmu(on_cmd_recieved) {
+ var self = this;
+ var welcome_msg = 'hi, im a varnish emu, right?';
-var assert = require('assert'),
- tests = module.exports = {},
- th = require(__dirname + '/../test_helper'),
- CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
+ self.commands_recieved = [];
-var cache = CacheValidator(global.environment.redis);
-
-tests["should get the timestamp it sets"] = function() {
- var d = new Date().getTime();
- cache.setTimestamp('mydatabase', 'mytable', d, function() {
- cache.getTimestamp('mydatabase', 'mytable', function(err, t) {
- assert.eql(t, d);
- })
+ var server = net.createServer(function (socket) {
+ var command = '';
+ socket.write("200 " + welcome_msg.length + "\n");
+ socket.write(welcome_msg);
+ socket.on('data', function(data) {
+ self.commands_recieved.push(data);
+ on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
+ socket.write('200 0\n');
+ });
});
+ server.listen(1337, "127.0.0.1");
}
-tests["should get null when timestamp is not set"] = function() {
- var d = new Date().getTime();
- cache.getTimestamp('mydatabase', 'mytable2', function(err, t) {
- assert.ok(t === null);
- })
+tests['working'] = function() { assert.ok(true); };
+
+tests['should call purge on varnish when invalidate database'] = function() {
+ var varnish = new VarnishEmu(function(cmds) {
+ assert.ok(cmds.length == 1);
+ assert.equal('purge obj.http.X-Cache-Channel == test_cache\n', cmds[0]);
+ });
+ CacheValidator.init('localhost', 1337);
+ CacheValidator.invalidate_db('test_cache');
}