Merge branch 'new_cache'

app.js

@@ -30,7 +30,6 @@ _.extend(global.settings, global.environment);
 
 var Windshaft = require('windshaft');
 var serverOptions = require('./lib/cartodb/server_options');
-var Cache       = require('./lib/cartodb/tile_cache');
 
 ws = CartodbWindshaft(serverOptions);
 ws.listen(global.environment.windshaft_port);

config/environments/development.js

@@ -5,7 +5,7 @@ module.exports.redis            = {host: '127.0.0.1',
                                    idleTimeoutMillis: 1,
                                    reapIntervalMillis: 1};
 module.exports.windshaft_port   = 8181;
-module.exports.lru_cache = true;
-module.exports.lru_cache_size = 10000;
 module.exports.enable_cors = true;
-module.exports.ttl_timeout = 5;
+module.exports.varnish_host = 'localhost';
+module.exports.varnish_port = 6082;
+module.exports.cache_enabled = true; 

config/environments/production.js

@@ -2,6 +2,7 @@ module.exports.name             = 'production';
 module.exports.postgres         = {user: 'tileuser', host: '127.0.0.1', port: 6432};
 module.exports.redis            = {host: '127.0.0.1', port: 6379};
 module.exports.windshaft_port   = 8181;
-module.exports.lru_cache = true;
-module.exports.lru_cache_size = 10000;
 module.exports.ttl_timeout = 600; // 10 minutes
+module.exports.varnish_host = 'localhost';
+module.exports.varnish_port = 6082
+module.exports.cache_enabled = true; 

config/environments/test.js

@@ -5,6 +5,7 @@ module.exports.redis            = {host: '127.0.0.1',
                                    idleTimeoutMillis: 1,
                                    reapIntervalMillis: 1};
 module.exports.windshaft_port   = 8080;
-module.exports.lru_cache = true;
-module.exports.lru_cache_size = 10000;
 module.exports.enable_cors = true;
+module.exports.varnish_host = '';
+module.exports.varnish_port = null;
+module.exports.cache_enabled = false;

lib/cartodb/cache_validator.js

@@ -1,76 +1,17 @@
-//
-// this module allows to invalidate cache for a table or check if the cache for a table
-// is valid
-//
-// usage:
-//
-// Cache.getTimestamp('table_name', function(err, timestamp) {
-// });
-//
-// Cache.setTimestamp('table_name', timestamp, function(err, result) {
-// });
-//
-//
-
 var _          = require('underscore'),
-    RedisPool  = require('./redis_pool'),
+    Varnish    = require('./varnish')
-    Step       = require('step')
 
+var varnish_queue = null;
 
-function Cache(redis_opts) {
+function init(host, port) {
-
+    varnish_queue = new Varnish.VarnishQueue(host, port);
-    var redis_pool = new RedisPool(redis_opts);
-
-    var me = {}
-
-    var redis_options = {
-        db: 0
 }
 
-    var redisCommand = function(fn, callback) {
+function invalidate_db(dbname) {
-        var redis_client;
+    varnish_queue.run_cmd('purge obj.http.X-Cache-Channel == ' + dbname);
-        Step(
-            function getRedisClient(){
-                redis_pool.acquire(redis_options.db, this);
-            },
-            function getDataForTable(err, redis) {
-                if (err) throw err;
-                redis_client = redis;
-                fn(redis_client, this);
-            },
-            function exit(err, data) {
-                if (!_.isUndefined(redis_client))
-                    redis_pool.release(redis_options.db, redis_client);
-                if (callback) {
-                    callback(err, data);
-                }
-            }
-        );
 }
 
-    // get the timestamp for the table
+module.exports = {
-    me.getTimestamp = function(database, table, callback) {
+    init: init,
-        redisCommand(function(redis, step) {
+    invalidate_db: invalidate_db
-            redis.GET("cache:" + database + ":" + table + ":last_updated_at", step);
-        }, function(err, t) {
-            if(t != null)
-                callback(err, parseFloat(t));
-            else
-                callback(err, t);
-        });
-    }
-
-    me.setTimestamp = function(database, table, timestamp, callback) {
-        timestamp = timestamp || new Date().getTime();
-        redisCommand(function(redis, step) {
-            redis.SET("cache:" + database + ":" + table + ":last_updated_at", timestamp, step);
-        }, callback);
-    }
-
-    return me;
-};
-
-
-module.exports = function(redis_opts) {
-    return new Cache(redis_opts);
 }

lib/cartodb/cartodb_windshaft.js

@@ -2,26 +2,30 @@
 var _ = require('underscore')
     , Step       = require('step')
     , Windshaft = require('windshaft')
-    , Cache       = require('./tile_cache');
+    , Cache = require('./cache_validator')
 
 var CartodbWindshaft = function(serverOptions) {
 
-    // set cache if requested
+    // set the cache chanel info to invalidate the cache on the frontend server
-    if(serverOptions.lru_cache) {
+    serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
-        var lru_cache = Cache.LRUcache(serverOptions.lru_cache_size || 10000,
+        res.header('X-Cache-Channel', req.params.dbname);
-                serverOptions.redis,
+        callback(null, tile, headers);
-                serverOptions.ttl_timeout);
+    };
-        _.extend(serverOptions, {
+
-            beforeTileRender: lru_cache.beforeTileRender,
+    if(serverOptions.cache_enabled) {
-            afterTileRender: lru_cache.afterTileRender,
+        console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
-            cacheStats: lru_cache.getStats,
+        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
-            afterStateChange: lru_cache.afterStateChange
+        serverOptions.afterStateChange = function(req, data, callback) {
-        })
+            Cache.invalidate_db(req.params.dbname);
+            callback(null, data);
         }
+    }
+
 
 
     // boot
     var ws = new Windshaft.Server(serverOptions);
+
     /**
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
@@ -40,6 +44,7 @@ var CartodbWindshaft = function(serverOptions) {
         );
     });
 
+
     /**
      * Helper to allow access to metadata to be used in embedded maps.
      */
@@ -58,17 +63,6 @@ var CartodbWindshaft = function(serverOptions) {
         );
     });
 
-    /**
-     * tile cache stats
-     */
-    ws.get('/cache', function(req, res){
-        if(serverOptions.cacheStats) {
-            res.send(serverOptions.cacheStats(req.query.tile_info, req.query.sort_by));
-        } else {
-            res.send("Cache no enabled")
-        }
-    });
-    
     return ws;
 }
 

lib/cartodb/lru.js

@@ -1,252 +0,0 @@
-/**
- * see original file: https://github.com/rsms/js-lru
- * 
- * A doubly linked list-based Least Recently Used (LRU) cache. Will keep most
- * recently used items while discarding least recently used items when its limit
- * is reached.
- *
- * Licensed under MIT. Copyright (c) 2010 Rasmus Andersson <http://hunch.se/>
- * See README.md for details.
- *
- * Illustration of the design:
- *
- *       entry             entry             entry             entry
- *       ______            ______            ______            ______
- *      | head |.newer => |      |.newer => |      |.newer => | tail |
- *      |  A   |          |  B   |          |  C   |          |  D   |
- *      |______| <= older.|______| <= older.|______| <= older.|______|
- *
- *  removed  <--  <--  <--  <--  <--  <--  <--  <--  <--  <--  <--  added
- */
-function LRUCache (limit) {
-  // Current size of the cache. (Read-only).
-  this.size = 0;
-  // Maximum number of items this cache can hold.
-  this.limit = limit;
-  this._keymap = {};
-}
-
-/**
- * Put <value> into the cache associated with <key>. Returns the entry which was
- * removed to make room for the new entry. Otherwise undefined is returned
- * (i.e. if there was enough room already).
- */
-LRUCache.prototype.put = function(key, value) {
-  var entry = {key:key, value:value};
-  // Note: No protection agains replacing, and thus orphan entries. By design.
-  this._keymap[key] = entry;
-  if (this.tail) {
-    // link previous tail to the new tail (entry)
-    this.tail.newer = entry;
-    entry.older = this.tail;
-  } else {
-    // we're first in -- yay
-    this.head = entry;
-  }
-  // add new entry to the end of the linked list -- it's now the freshest entry.
-  this.tail = entry;
-  if (this.size === this.limit) {
-    // we hit the limit -- remove the head
-    return this.shift();
-  } else {
-    // increase the size counter
-    this.size++;
-  }
-}
-
-/**
- * Purge the least recently used (oldest) entry from the cache. Returns the
- * removed entry or undefined if the cache was empty.
- *
- * If you need to perform any form of finalization of purged items, this is a
- * good place to do it. Simply override/replace this function:
- *
- *   var c = new LRUCache(123);
- *   c.shift = function() {
- *     var entry = LRUCache.prototype.shift.call(this);
- *     doSomethingWith(entry);
- *     return entry;
- *   }
- */
-LRUCache.prototype.shift = function() {
-  // todo: handle special case when limit == 1
-  var entry = this.head;
-  if (entry) {
-    if (this.head.newer) {
-      this.head = this.head.newer;
-      this.head.older = undefined;
-    } else {
-      this.head = undefined;
-    }
-    // Remove last strong reference to <entry> and remove links from the purged
-    // entry being returned:
-    entry.newer = entry.older = undefined;
-    // delete is slow, but we need to do this to avoid uncontrollable growth:
-    delete this._keymap[entry.key];
-  }
-  return entry;
-}
-
-/**
- * Get and register recent use of <key>. Returns the value associated with <key>
- * or undefined if not in cache.
- */
-LRUCache.prototype.get = function(key, returnEntry) {
-  // First, find our cache entry
-  var entry = this._keymap[key];
-  if (entry === undefined) return; // Not cached. Sorry.
-  // As <key> was found in the cache, register it as being requested recently
-  if (entry === this.tail) {
-    // Already the most recenlty used entry, so no need to update the list
-    return entry.value;
-  }
-  // HEAD--------------TAIL
-  //   <.older   .newer>
-  //  <--- add direction --
-  //   A  B  C  <D>  E
-  if (entry.newer) {
-    if (entry === this.head)
-      this.head = entry.newer;
-    entry.newer.older = entry.older; // C <-- E.
-  }
-  if (entry.older)
-    entry.older.newer = entry.newer; // C. --> E
-  entry.newer = undefined; // D --x
-  entry.older = this.tail; // D. --> E
-  if (this.tail)
-    this.tail.newer = entry; // E. <-- D
-  this.tail = entry;
-  return returnEntry ? entry : entry.value;
-}
-
-// ----------------------------------------------------------------------------
-// Following code is optional and can be removed without breaking the core
-// functionality.
-
-/**
- * Check if <key> is in the cache without registering recent use. Feasible if
- * you do not want to chage the state of the cache, but only "peek" at it.
- * Returns the entry associated with <key> if found, or undefined if not found.
- */
-LRUCache.prototype.find = function(key) {
-  return this._keymap[key];
-}
-
-/**
- * Update the value of entry with <key>. Returns the old value, or undefined if
- * entry was not in the cache.
- */
-LRUCache.prototype.set = function(key, value) {
-  var oldvalue, entry = this.get(key, true);
-  if (entry) {
-    oldvalue = entry.value;
-    entry.value = value;
-  } else {
-    oldvalue = this.put(key, value);
-    if (oldvalue) oldvalue = oldvalue.value;
-  }
-  return oldvalue;
-}
-
-/**
- * Remove entry <key> from cache and return its value. Returns undefined if not
- * found.
- */
-LRUCache.prototype.remove = function(key) {
-  var entry = this._keymap[key];
-  if (!entry) return;
-  delete this._keymap[entry.key]; // need to do delete unfortunately
-  if (entry.newer && entry.older) {
-    // relink the older entry with the newer entry
-    entry.older.newer = entry.newer;
-    entry.newer.older = entry.older;
-  } else if (entry.newer) {
-    // remove the link to us
-    entry.newer.older = undefined;
-    // link the newer entry to head
-    this.head = entry.newer;
-  } else if (entry.older) {
-    // remove the link to us
-    entry.older.newer = undefined;
-    // link the newer entry to head
-    this.tail = entry.older;
-  } else { 
-    this.head = this.tail = undefined;
-  }
-  this.size--;
-  return entry.value;
-}
-
-/** Removes all entries */
-LRUCache.prototype.removeAll = function() {
-  // This should be safe, as we never expose strong refrences to the outside
-  this.head = this.tail = undefined;
-  this.size = 0;
-  this._keymap = {};
-}
-
-/**
- * Return an array containing all keys of entries stored in the cache object, in
- * arbitrary order.
- */
-if (typeof Object.keys === 'function') {
-  LRUCache.prototype.keys = function() { return Object.keys(this._keymap); }
-} else {
-  LRUCache.prototype.keys = function() {
-    var keys = [];
-    for (var k in this._keymap) keys.push(k);
-    return keys;
-  }
-}
-
-/**
- * Call `fun` for each entry. Starting with the newest entry if `desc` is a true
- * value, otherwise starts with the oldest (head) enrty and moves towards the
- * tail.
- *
- * `fun` is called with 3 arguments in the context `context`:
- *   `fun.call(context, Object key, Object value, LRUCache self)`
- */
-LRUCache.prototype.forEach = function(fun, context, desc) {
-  if (context === true) { desc = true; context = undefined; }
-  else if (typeof context !== 'object') context = this;
-  if (desc) {
-    var entry = this.tail;
-    while (entry) {
-      fun.call(context, entry.key, entry.value, this);
-      entry = entry.older;
-    }
-  } else {
-    var entry = this.head;
-    while (entry) {
-      fun.call(context, entry.key, entry.value, this);
-      entry = entry.newer;
-    }
-  }
-}
-
-/** Returns a JSON (array) representation */
-LRUCache.prototype.toJSON = function() {
-  var s = [], entry = this.head;
-  while (entry) {
-    s.push({key:entry.key.toJSON(), value:entry.value.toJSON()});
-    entry = entry.newer;
-  }
-  return s;
-}
-
-/** Returns a String representation */
-LRUCache.prototype.toString = function() {
-  var s = '', entry = this.head;
-  while (entry) {
-    s += String(entry.key)+':'+entry.value;
-    if (entry = entry.newer)
-      s += ' < ';
-  }
-  return s;
-}
-
-// Export ourselves
-//if (typeof this === 'object') this.LRUCache = LRUCache;
-
-module.exports = LRUCache;

lib/cartodb/server_options.js

@@ -8,9 +8,9 @@ module.exports = function(){
         grainstore: {datasource: global.environment.postgres},
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
-        lru_cache: global.environment.lru_cache,
+        varnish_host: global.environment.varnish_host,
-        lru_cache_size: global.environment.lru_cache_size,
+        varnish_port: global.environment.varnish_port,
-        ttl_timeout: global.environment.ttl_timeout
+        cache_enabled: global.environment.cache_enabled
 };
 
 /**

lib/cartodb/tile_cache.js

@@ -1,178 +0,0 @@
-// tile cache policies
-// it exports two cache types:
-// 'nocache' implements a pass-troguth cache
-// 'lru' implements a LRU cache
-
-var LRUCache       = require('./lru'),
-    CacheValidator = require('./cache_validator'),
-    TTL            = require('./ttl');
-
-var BLANK_TILE_SIZE = 334; // totally transparent tile size in bytes
-
-module.exports.NoCache = function() {
-
-    var me = {}
-
-    me.beforeTileRender = function(req, res, callback) {
-        callback(null);
-    }
-
-    me.afterTileRender = function(req, res, tile, headers, callback) {
-        callback(null, tile, headers);
-    }
-
-    return me;
-
-}
-
-module.exports.LRUcache = function(max_items, redis_opts, ttl_timeout) {
-    var cache_validator = CacheValidator(redis_opts);
-    return GenericCache(new LRUCache(max_items), cache_validator, ttl_timeout || 600);
-}
-
-// implements a generic cache for tile
-// cache_policy should implement set and get methods and optionally getStats
-function GenericCache (cache_policy, cache_validator, ttl_timeout) {
-
-    var me = {
-        cache: cache_policy,
-        cache_validator: cache_validator,
-        cache_hits: 0,
-        cache_misses: 0,
-        cache_not_modified: 0,
-        current_items: 0,
-        expired: 0,
-        cache_invalidated: 0,
-        max_items: 0
-    };
-
-    // enable ttl
-    var ttl = TTL(function(key) {
-        me.remove(key);
-        me.expired++;
-    }, ttl_timeout || 60);
-
-    function cache_key(req) {
-        return req.url;
-    }
-
-    function update_items(n) {
-        me.current_items = n;
-        if(n > me.max_items) {
-            me.max_items = n;
-        }
-    }
-
-    me.remove = function(key) {
-        ttl.remove(key);
-        me.cache.remove(key);
-        update_items(me.cache.size || 0);
-    }
-
-    me.beforeTileRender = function(req, res, callback) {
-        req.windshaft_start = new Date().getTime();
-        var key = cache_key(req);
-        var tile = me.cache.get(key);
-        if(tile) {
-            // validate the cache
-            me.cache_validator.getTimestamp(req.params.dbname, req.params.table, function(err, t) {
-                if(t !== null && tile.timestamp < t) {
-                    me.cache_misses++;
-                    me.cache_invalidated++;
-                    callback(null);
-                } else {
-                    // stats
-                    me.cache_hits++;
-                    var timestamp = new Date().getTime();
-                    var delta = timestamp - req.windshaft_start;
-                    tile.cache_time = delta/1000.0;
-                    tile.hits++;
-                    res.header('X-Cache-hit', 'true');
-                    res.header('Last-Modified', new Date(tile.timestamp*1000).toGMTString());
-                    // check 304
-                    var modified_since = req.header('if-modified-since');
-                    if (modified_since) {
-                        modified_since = Date.parse(modified_since);
-                        if(modified_since && modified_since <= timestamp*1000) {
-                            me.cache_not_modified++;
-                            res.statusCode = 304;
-                            res.end();
-                            return;
-                        }
-                    }
-                    res.send(tile.tile, tile.headers, 200);
-                }
-            });
-        } else {
-            me.cache_misses++;
-            callback(null);
-        }
-    }
-
-    me.afterTileRender = function(req, res, tile, headers, callback) {
-        var timestamp = new Date().getTime();
-        var delta = timestamp - req.windshaft_start;
-        var key = cache_key(req);
-        me.cache.put(key, {
-                tile: tile,
-                headers: headers,
-                timestamp: timestamp/1000.0,
-                render_time: delta/1000.0,
-                hits: 0});
-        ttl.start(key);
-        update_items(me.cache.size || 0);
-        callback(null, tile, headers);
-    }
-
-    me.afterStateChange = function(req, data, callback) {
-        me.cache_validator.setTimestamp(req.params.dbname, req.params.table, new Date().getTime()/1000.0, function(err, t) {
-            callback(err, data);
-        });
-    }
-
-    me.getStats = function(include_tile_info, sort_by) {
-        var total = me.cache_hits + me.cache_misses;
-        var mem = 0;
-        var blank_tile_count = 0;
-        var tile_info = []
-        me.cache.forEach(function(key, value) {
-            if(value.tile.length !== undefined) {
-                mem += value.tile.length;
-                if(value.tile.length == BLANK_TILE_SIZE) {
-                    blank_tile_count++;
-                }
-            }
-            if(include_tile_info) {
-                tile_info.push({
-                    key: key,
-                    length: value.tile.length,
-                    hits: value.hits,
-                    render_time: value.render_time,
-                    cache_time: value.cache_time
-                });
-            }
-        });
-        sort_by = sort_by || 'hits';
-        tile_info.sort(function(a, b) {
-            return b[sort_by] - a[sort_by];
-        });
-        return  {
-             cache_hits: me.cache_hits,
-             cache_misses: me.cache_misses,
-             cache_invalidated: me.cache_invalidated,
-             cache_not_modified: me.cache_not_modified,
-             current_items: me.current_items,
-             expired: me.expired,
-             max_items: me.max_items,
-             memory: mem,
-             memory_per_item: total ? mem/total: 0,
-             ratio: total ? me.cache_hits/total: 0,
-             blank_tile_count: blank_tile_count,
-             blank_tile_size: blank_tile_count*BLANK_TILE_SIZE,
-             blank_items_ratio: total? blank_tile_count/total: 0,
-             tiles: tile_info
-        };
-    }
-
-   return me;
-}

lib/cartodb/ttl.js

@@ -1,47 +0,0 @@
-
-/*
- ==========================================================
- manages timeouts
-     * timeout in seconds
-     * on_delete will be called when time expired with the key as first param
-  
-  usage:
-    ttl = TTL(function(key) {
-        console.log("10 seconds expired on " + key");
-    }, 10);
- ==========================================================
-*/
-function TTL(on_delete, timeout) {
-
-    var me = {
-        timeouts: {},
-        on_delete: on_delete,
-        timeout: timeout*1000
-    };
-
-    me.start = function(key) {
-        var t = me.timeouts[key];
-        if (t) {
-            clearTimeout(t);
-        }
-        me.timeouts[key] = setTimeout(
-            (function(k) {
-                return function() {
-                    me.on_delete(k);
-                };
-            })(key),
-            me.timeout);
-    }
-
-    me.remove = function(key) {
-        var t = me.timeouts[key];
-        if (t) {
-           clearTimeout(t);
-           delete me.timeouts[key];
-        }
-    }
-
-    return me;
-}
-
-module.exports = TTL;

lib/cartodb/varnish.js

@@ -0,0 +1,180 @@
+
+/**
+ * this module implements varnish telnet management protocol
+ * https://www.varnish-cache.org/trac/wiki/ManagementPort
+ */
+
+var net = require('net')
+
+function VarnishClient(host, port, ready_callback) {
+
+    var self = this;
+    var ready = false;
+    var cmd_callback = null;
+    var client = null;
+
+    function connect() {
+        ready = false;
+        client = net.createConnection(port, host);
+    }
+
+    connect();
+
+    client.on('data', function(data) {
+        data = data.toString();
+        lines = data.split('\n', 2);
+        if(lines.length == 2) {
+            var tk = lines[0].split(' ')
+            var code = parseInt(tk[0], 10);
+            var body_length = parseInt(tk[1], 10);
+            var body = lines[1];
+            if(!ready) {
+                ready = true;
+                ready_callback();
+            } else if(cmd_callback) {
+                var c = cmd_callback
+                cmd_callback = null;
+                c(null, code, body);
+            }
+        }
+
+    });
+
+    client.on('error', function(err) {
+        console.log("[ERROR] some problem in varnish connection");
+    });
+
+    client.on('close', function() {
+        self.close();
+    });
+
+    // sends the command to the server
+    function _send(cmd, callback) {
+      cmd_callback = callback;
+      client.write(cmd + '\n');
+    }
+
+    // run command if there is no peding response
+    // fist param of the callback are the error, null
+    // if all went ok
+    this.run_cmd = function(cmd, callback) {
+       if(!cmd_callback) {
+         _send(cmd, callback);
+       } else {
+         callback('response pending');
+       }
+    }
+
+    // close the connection
+    this.close = function() {
+       client.end();
+       ready = false; 
+    }
+
+}
+
+
+function VarnishPool(opts, ready) {
+    var resources = [];
+    var available = [];
+
+    for(var i = 0; i < opts.pool_size; ++i) {
+        var v = new VarnishClient(opts.host, opts.port, function() {
+            resources.push(v);
+            available.push(v);
+        });
+    }
+
+    this.run_cmd = function(cmd, callback) {
+        var v = available.pop()
+        if(v) {
+            v.run_cmd(cmd, function(err, status_code, body) {
+                callback(err, status_code, body);
+                available.push(v);
+                ready();
+            });
+        } else {
+            callback('no clients available');
+        }
+    }
+
+    this.close = function() {
+        for(var i = 0; i < resources.length; ++i) {
+            resources[i].close();
+        }
+    }
+}
+
+function VarnishQueue(host, port) {
+
+    var MAX_QUEUE = 2000;
+    var queue = [];
+    var ready = false;
+
+    var client = new VarnishClient(host, port, function(err) {
+        ready = true;
+        _send_pending();
+    });
+
+    function _send_pending(empty_callback) {
+        var c = queue.pop();
+        if(!c) return;
+        client.run_cmd(c, function() {
+            if(queue.length > 0) {
+                process.nextTick(_send_pending);
+            } else {
+                if(empty_callback) {
+                    empty_callback();
+                }
+            }
+        });
+    }
+
+    this.run_cmd = function(cmd) {
+        queue.push(cmd);
+        if(queue.length > MAX_QUEUE) {
+            console.log("varnish commando queu too long, removing commands");
+            queue.pop();
+        }
+        if(ready) {
+            _send_pending();
+        }
+    }
+
+    this.end = function() {
+        _send_pending(function() {
+            client.close();
+        });
+    }
+
+}
+
+/*
+var queue = new VarnishQueue('localhost', 6082)
+setInterval(function() {
+    queue.run_cmd('purge obj.http.url == /')
+}, 10)
+*/
+/*
+v = new VarnishClient('localhost', 6082, function(err) {
+    console.log('connected');
+    v.run_cmd('purge obj.http.url == /', function(err, code, body) {
+        console.log(code);
+        console.log(body);
+        v.close();
+    });
+});
+
+pool = new VarnishPool({
+    host: 'locahost',
+    port: 6082,
+    pool_size: 5
+});
+/*
+v.close();
+*/
+
+module.exports = {
+    VarnishClient: VarnishClient,
+    VarnishQueue: VarnishQueue
+}

test/acceptance/cache.js

@@ -1,214 +0,0 @@
-
-var   assert      = require('assert')
-    , tests       = module.exports = {}
-    , _           = require('underscore')
-    , querystring = require('querystring')
-    , fs          = require('fs')
-    , th          = require(__dirname + '/../test_helper')
-    , CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft')
-    , http          = require('http')
-    , Step          = require('step')
-    , CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
-
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
-
-var cached_server = new CartodbWindshaft(serverOptions);
-
-tests["first time a tile is request should not be cached"] = function() {
-    assert.response(cached_server, {
-        url: '/tiles/test_table_3/6/31/24.png?geom_type=polygon',
-        headers: {host: 'vizzuality.localhost.lan'},
-        method: 'GET'
-    },{
-        status: 200
-
-    }, function(res) {
-        assert.ok(res.header('X-Cache-hit') === undefined);
-
-    });
-
-}
-
-/*
-tests["second time a tile is request should be cached"] = function() {
-
-   var cached_server2 = new CartodbWindshaft(serverOptions);
-   var url= '/tiles/test_table_2/6/31/24.png';
-   assert.response(cached_server2, {
-            url: url,
-            headers: {host: 'vizzuality.localhost.lan'},
-            method: 'GET'
-    },{
-        status: 200
-    }, function(res) {
-        assert.response(cached_server2, {
-            url: url,
-            headers: {host: 'vizzuality.localhost.lan'},
-            method: 'GET'
-        },{
-            status: 200
-        }, function(res) {
-            assert.ok(res.header('X-Cache-hit') !== undefined);
-        });
-    });
-}
-*/
-
-
-tests["LRU tile should be removed"] = function() {
-
-    var urls = ['/tiles/test_table_2/6/31/24.png',
-               '/tiles/test_table_2/6/31/25.png',
-               '/tiles/test_table_2/6/31/26.png',
-               '/tiles/test_table_2/6/31/27.png'];
-    
-    //create another server to not take previos test stats into account
-    var so = _.clone(serverOptions);
-    _(so).extend({lru_cache: true, lru_cache_size: 3});
-
-    var _cached_server = new CartodbWindshaft(so);
-    
-    function makeReq(url, callback) {
-        assert.response(_cached_server, {
-                url: url,
-                headers: {host: 'vizzuality.localhost.lan'},
-                method: 'GET'
-        },{
-            status: 200
-        }, callback);
-    }
-
-    Step(
-        function() {
-            makeReq(urls[0], this);
-        },
-        function() {
-            makeReq(urls[1], this);
-        },
-        function() {
-            makeReq(urls[2], this);
-        },
-         function() {
-            makeReq(urls[3], this);
-        }, function() {
-            assert.response(_cached_server, {
-            url: urls[0],
-            headers: {host: 'vizzuality.localhost.lan'},
-            method: 'GET'
-            },{
-                status: 200
-
-            }, function(res) {
-                assert.ok(res.header('X-Cache-hit') === undefined);
-                var st = so.cacheStats()
-                assert.eql(st.cache_hits, 0);
-                assert.eql(st.cache_misses, 5);
-                assert.eql(st.current_items, 3);
-            });
-        }
-    )
-
-}
-
-tests["cache should be invalidated"] = function() {
-
-   var url = '/tiles/test_table_2/6/29/27.png';
-   var cached_server2 = new CartodbWindshaft(serverOptions);
-   var cache = CacheValidator(global.environment.redis);
-   assert.response(cached_server2, {
-            url: url,
-            headers: {host: 'vizzuality.localhost.lan'},
-            method: 'GET'
-    },{
-        status: 200
-    }, function(res) {
-        cache.setTimestamp('cartodb_test_user_1_db', 'test_table_2', (new Date().getTime()/1000.0)+100, function() {
-            assert.response(cached_server2, {
-                url: url,
-                headers: {host: 'vizzuality.localhost.lan'},
-                method: 'GET'
-            },{
-                status: 200
-            }, function(r) {
-                assert.ok(r.header('X-Cache-hit') === undefined);
-            });
-        });
-    });
-
-};
-
-tests["Last-Modified header should be sent"] = function() {
-   var cached_server2 = new CartodbWindshaft(serverOptions);
-   var url= '/tiles/test_table/6/31/22.png';
-   assert.response(cached_server2, {
-            url: url,
-            headers: {host: 'vizzuality.localhost.lan'},
-            method: 'GET'
-    },{
-        status: 200
-    }, function(res) {
-        assert.response(cached_server2, {
-            url: url,
-            headers: {host: 'vizzuality.localhost.lan'},
-            method: 'GET'
-        },{
-           status: 200
-        }, function(res) {
-            assert.ok(res.header('X-Cache-hit') !== undefined);
-            var last_modified = res.header('Last-Modified');
-            assert.ok(last_modified !== undefined);
-            assert.response(cached_server2, {
-                    url: url,
-                    headers: {
-                        host: 'vizzuality.localhost.lan',
-                        'if-modified-since': last_modified
-                    },
-                    method: 'GET'
-           }, {
-                status: 304
-           });
-        });
-    });
-}
-
-tests["TTL should invalidate a tile"] = function() {
-
-    var url = '/tiles/test_table_2/6/31/24.png';
-    
-    //create another server to not take previos test stats into account
-    var so = _.clone(serverOptions);
-    _(so).extend({ttl_timeout: 1});
-
-    var _cached_server = new CartodbWindshaft(so);
-    // cache it
-    assert.response(_cached_server, {
-            url: url,
-            headers: {host: 'vizzuality.localhost.lan'},
-            method: 'GET'
-    },{
-        status: 200
-    }, function(res) {
-        console.log("WAIT A LITTLE BIT PLEASE");
-
-        // test before invalidating
-        setTimeout(function() {
-            var st = so.cacheStats();
-            assert.eql(st.expired, 0);
-        }, 500);
-
-        // test after invalidation
-        setTimeout(function() {
-            assert.response(_cached_server, {
-                    url: url,
-                    headers: {host: 'vizzuality.localhost.lan'},
-                    method: 'GET'
-            },{
-                status: 200
-            }, function(res) {
-                assert.ok(res.header('X-Cache-hit') === undefined);
-                var st = so.cacheStats();
-                assert.eql(st.expired, 1);
-            });
-        }, 2000);
-    });
-}

test/acceptance/cache_validator.js

@@ -1,24 +1,35 @@
+var assert      = require('assert');
+var net = require('net');
+require(__dirname + '/../test_helper');
+var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator');
+var tests = module.exports = {};
 
+function VarnishEmu(on_cmd_recieved) {
+    var self = this;
+    var welcome_msg = 'hi, im a varnish emu, right?';
 
-var   assert      = require('assert'),
+    self.commands_recieved = [];
-      tests       = module.exports = {},
-      th          = require(__dirname + '/../test_helper'),
-      CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
 
-var cache = CacheValidator(global.environment.redis);
+    var server = net.createServer(function (socket) {
-
+      var command = '';
-tests["should get the timestamp it sets"] = function() {
+      socket.write("200 " + welcome_msg.length + "\n");
-    var d = new Date().getTime();
+      socket.write(welcome_msg);
-    cache.setTimestamp('mydatabase', 'mytable', d, function() {
+      socket.on('data', function(data) {
-        cache.getTimestamp('mydatabase', 'mytable', function(err, t) {
+        self.commands_recieved.push(data);
-            assert.eql(t, d);
+        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
-        })
+        socket.write('200 0\n');
       });
+    });
+    server.listen(1337, "127.0.0.1");
 }
 
-tests["should get null when timestamp is not set"] = function() {
+tests['working'] = function() { assert.ok(true); };
-    var d = new Date().getTime();
+
-    cache.getTimestamp('mydatabase', 'mytable2', function(err, t) {
+tests['should call purge on varnish when invalidate database'] = function() {
-        assert.ok(t === null);
+    var varnish = new VarnishEmu(function(cmds) {
-    })
+        assert.ok(cmds.length == 1);
+        assert.equal('purge obj.http.X-Cache-Channel == test_cache\n', cmds[0]);
+    });
+    CacheValidator.init('localhost', 1337);
+    CacheValidator.invalidate_db('test_cache');
 }