From cb69faffc0d9ea1aed24e3fbae95eeae76658751 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 29 Nov 2011 21:19:10 +0100 Subject: [PATCH 1/7] added X-Cache-Channel header so the tile cache could be invalidated using that key for the moment we're using the database name as cache key, it is not the best solution because all the tiles for that database will be invalidated when a change is done in any table. Doing this we avoid getting wrong data when user gets tiles which come from a query with joins. --- lib/cartodb/cartodb_windshaft.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 195f00b6..568fc002 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -16,12 +16,18 @@ var CartodbWindshaft = function(serverOptions) { afterTileRender: lru_cache.afterTileRender, cacheStats: lru_cache.getStats, afterStateChange: lru_cache.afterStateChange - }) + }); } + // set the cache chanel info to invalidate the cache on the frontend server + serverOptions.afterTileRender = function(req, res, tile, headers, callback) { + res.header('X-Cache-Channel', req.params.dbname); + callback(null, tile, headers); + }; // boot var ws = new Windshaft.Server(serverOptions); + /** * Helper to allow access to the layer to be used in the maps infowindow popup. */ From 472ed8bd7907248856a967aef60f7f7ef172bac1 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 29 Nov 2011 21:19:20 +0100 Subject: [PATCH 2/7] disabled caching --- config/environments/development.js | 2 +- config/environments/production.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/environments/development.js b/config/environments/development.js index e02b1c66..ed8674cb 100644 --- a/config/environments/development.js +++ b/config/environments/development.js @@ -5,7 +5,7 @@ module.exports.redis = {host: '127.0.0.1', idleTimeoutMillis: 1, reapIntervalMillis: 1}; module.exports.windshaft_port = 8181; -module.exports.lru_cache = true; +module.exports.lru_cache = false; module.exports.lru_cache_size = 10000; module.exports.enable_cors = true; module.exports.ttl_timeout = 5; diff --git a/config/environments/production.js b/config/environments/production.js index d1ed9634..5d4bdaad 100644 --- a/config/environments/production.js +++ b/config/environments/production.js @@ -2,6 +2,6 @@ module.exports.name = 'production'; module.exports.postgres = {user: 'tileuser', host: '127.0.0.1', port: 6432, max_size: 4}; module.exports.redis = {host: '127.0.0.1', port: 6379}; module.exports.windshaft_port = 8181; -module.exports.lru_cache = true; +module.exports.lru_cache = false; module.exports.lru_cache_size = 10000; module.exports.ttl_timeout = 600; // 10 minutes From 54df010694ea58a0ded5e8881dd779f822b09f6f Mon Sep 17 00:00:00 2001 From: javi santana Date: Wed, 30 Nov 2011 19:57:26 +0100 Subject: [PATCH 3/7] removed old cache code --- lib/cartodb/lru.js | 252 ----------------------------- lib/cartodb/tile_cache.js | 178 -------------------- lib/cartodb/ttl.js | 47 ------ test/acceptance/cache.js | 214 ------------------------ test/acceptance/cache_validator.js | 24 --- 5 files changed, 715 deletions(-) delete mode 100644 lib/cartodb/lru.js delete mode 100644 lib/cartodb/tile_cache.js delete mode 100644 lib/cartodb/ttl.js delete mode 100644 test/acceptance/cache.js delete mode 100644 test/acceptance/cache_validator.js diff --git a/lib/cartodb/lru.js b/lib/cartodb/lru.js deleted file mode 100644 index adb8a8f5..00000000 --- a/lib/cartodb/lru.js +++ /dev/null @@ -1,252 +0,0 @@ -/** - * see original file: https://github.com/rsms/js-lru - * - * A doubly linked list-based Least Recently Used (LRU) cache. Will keep most - * recently used items while discarding least recently used items when its limit - * is reached. - * - * Licensed under MIT. Copyright (c) 2010 Rasmus Andersson - * See README.md for details. - * - * Illustration of the design: - * - * entry entry entry entry - * ______ ______ ______ ______ - * | head |.newer => | |.newer => | |.newer => | tail | - * | A | | B | | C | | D | - * |______| <= older.|______| <= older.|______| <= older.|______| - * - * removed <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- added - */ -function LRUCache (limit) { - // Current size of the cache. (Read-only). - this.size = 0; - // Maximum number of items this cache can hold. - this.limit = limit; - this._keymap = {}; -} - -/** - * Put into the cache associated with . Returns the entry which was - * removed to make room for the new entry. Otherwise undefined is returned - * (i.e. if there was enough room already). - */ -LRUCache.prototype.put = function(key, value) { - var entry = {key:key, value:value}; - // Note: No protection agains replacing, and thus orphan entries. By design. - this._keymap[key] = entry; - if (this.tail) { - // link previous tail to the new tail (entry) - this.tail.newer = entry; - entry.older = this.tail; - } else { - // we're first in -- yay - this.head = entry; - } - // add new entry to the end of the linked list -- it's now the freshest entry. - this.tail = entry; - if (this.size === this.limit) { - // we hit the limit -- remove the head - return this.shift(); - } else { - // increase the size counter - this.size++; - } -} - -/** - * Purge the least recently used (oldest) entry from the cache. Returns the - * removed entry or undefined if the cache was empty. - * - * If you need to perform any form of finalization of purged items, this is a - * good place to do it. Simply override/replace this function: - * - * var c = new LRUCache(123); - * c.shift = function() { - * var entry = LRUCache.prototype.shift.call(this); - * doSomethingWith(entry); - * return entry; - * } - */ -LRUCache.prototype.shift = function() { - // todo: handle special case when limit == 1 - var entry = this.head; - if (entry) { - if (this.head.newer) { - this.head = this.head.newer; - this.head.older = undefined; - } else { - this.head = undefined; - } - // Remove last strong reference to and remove links from the purged - // entry being returned: - entry.newer = entry.older = undefined; - // delete is slow, but we need to do this to avoid uncontrollable growth: - delete this._keymap[entry.key]; - } - return entry; -} - -/** - * Get and register recent use of . Returns the value associated with - * or undefined if not in cache. - */ -LRUCache.prototype.get = function(key, returnEntry) { - // First, find our cache entry - var entry = this._keymap[key]; - if (entry === undefined) return; // Not cached. Sorry. - // As was found in the cache, register it as being requested recently - if (entry === this.tail) { - // Already the most recenlty used entry, so no need to update the list - return entry.value; - } - // HEAD--------------TAIL - // <.older .newer> - // <--- add direction -- - // A B C E - if (entry.newer) { - if (entry === this.head) - this.head = entry.newer; - entry.newer.older = entry.older; // C <-- E. - } - if (entry.older) - entry.older.newer = entry.newer; // C. --> E - entry.newer = undefined; // D --x - entry.older = this.tail; // D. --> E - if (this.tail) - this.tail.newer = entry; // E. <-- D - this.tail = entry; - return returnEntry ? entry : entry.value; -} - -// ---------------------------------------------------------------------------- -// Following code is optional and can be removed without breaking the core -// functionality. - -/** - * Check if is in the cache without registering recent use. Feasible if - * you do not want to chage the state of the cache, but only "peek" at it. - * Returns the entry associated with if found, or undefined if not found. - */ -LRUCache.prototype.find = function(key) { - return this._keymap[key]; -} - -/** - * Update the value of entry with . Returns the old value, or undefined if - * entry was not in the cache. - */ -LRUCache.prototype.set = function(key, value) { - var oldvalue, entry = this.get(key, true); - if (entry) { - oldvalue = entry.value; - entry.value = value; - } else { - oldvalue = this.put(key, value); - if (oldvalue) oldvalue = oldvalue.value; - } - return oldvalue; -} - -/** - * Remove entry from cache and return its value. Returns undefined if not - * found. - */ -LRUCache.prototype.remove = function(key) { - var entry = this._keymap[key]; - if (!entry) return; - delete this._keymap[entry.key]; // need to do delete unfortunately - if (entry.newer && entry.older) { - // relink the older entry with the newer entry - entry.older.newer = entry.newer; - entry.newer.older = entry.older; - } else if (entry.newer) { - // remove the link to us - entry.newer.older = undefined; - // link the newer entry to head - this.head = entry.newer; - } else if (entry.older) { - // remove the link to us - entry.older.newer = undefined; - // link the newer entry to head - this.tail = entry.older; - } else { - this.head = this.tail = undefined; - } - this.size--; - return entry.value; -} - -/** Removes all entries */ -LRUCache.prototype.removeAll = function() { - // This should be safe, as we never expose strong refrences to the outside - this.head = this.tail = undefined; - this.size = 0; - this._keymap = {}; -} - -/** - * Return an array containing all keys of entries stored in the cache object, in - * arbitrary order. - */ -if (typeof Object.keys === 'function') { - LRUCache.prototype.keys = function() { return Object.keys(this._keymap); } -} else { - LRUCache.prototype.keys = function() { - var keys = []; - for (var k in this._keymap) keys.push(k); - return keys; - } -} - -/** - * Call `fun` for each entry. Starting with the newest entry if `desc` is a true - * value, otherwise starts with the oldest (head) enrty and moves towards the - * tail. - * - * `fun` is called with 3 arguments in the context `context`: - * `fun.call(context, Object key, Object value, LRUCache self)` - */ -LRUCache.prototype.forEach = function(fun, context, desc) { - if (context === true) { desc = true; context = undefined; } - else if (typeof context !== 'object') context = this; - if (desc) { - var entry = this.tail; - while (entry) { - fun.call(context, entry.key, entry.value, this); - entry = entry.older; - } - } else { - var entry = this.head; - while (entry) { - fun.call(context, entry.key, entry.value, this); - entry = entry.newer; - } - } -} - -/** Returns a JSON (array) representation */ -LRUCache.prototype.toJSON = function() { - var s = [], entry = this.head; - while (entry) { - s.push({key:entry.key.toJSON(), value:entry.value.toJSON()}); - entry = entry.newer; - } - return s; -} - -/** Returns a String representation */ -LRUCache.prototype.toString = function() { - var s = '', entry = this.head; - while (entry) { - s += String(entry.key)+':'+entry.value; - if (entry = entry.newer) - s += ' < '; - } - return s; -} - -// Export ourselves -//if (typeof this === 'object') this.LRUCache = LRUCache; - -module.exports = LRUCache; diff --git a/lib/cartodb/tile_cache.js b/lib/cartodb/tile_cache.js deleted file mode 100644 index 92b1737e..00000000 --- a/lib/cartodb/tile_cache.js +++ /dev/null @@ -1,178 +0,0 @@ -// tile cache policies -// it exports two cache types: -// 'nocache' implements a pass-troguth cache -// 'lru' implements a LRU cache - -var LRUCache = require('./lru'), - CacheValidator = require('./cache_validator'), - TTL = require('./ttl'); - -var BLANK_TILE_SIZE = 334; // totally transparent tile size in bytes - -module.exports.NoCache = function() { - - var me = {} - - me.beforeTileRender = function(req, res, callback) { - callback(null); - } - - me.afterTileRender = function(req, res, tile, headers, callback) { - callback(null, tile, headers); - } - - return me; - -} - -module.exports.LRUcache = function(max_items, redis_opts, ttl_timeout) { - var cache_validator = CacheValidator(redis_opts); - return GenericCache(new LRUCache(max_items), cache_validator, ttl_timeout || 600); -} - -// implements a generic cache for tile -// cache_policy should implement set and get methods and optionally getStats -function GenericCache (cache_policy, cache_validator, ttl_timeout) { - - var me = { - cache: cache_policy, - cache_validator: cache_validator, - cache_hits: 0, - cache_misses: 0, - cache_not_modified: 0, - current_items: 0, - expired: 0, - cache_invalidated: 0, - max_items: 0 - }; - - // enable ttl - var ttl = TTL(function(key) { - me.remove(key); - me.expired++; - }, ttl_timeout || 60); - - function cache_key(req) { - return req.url; - } - - function update_items(n) { - me.current_items = n; - if(n > me.max_items) { - me.max_items = n; - } - } - - me.remove = function(key) { - ttl.remove(key); - me.cache.remove(key); - update_items(me.cache.size || 0); - } - - me.beforeTileRender = function(req, res, callback) { - req.windshaft_start = new Date().getTime(); - var key = cache_key(req); - var tile = me.cache.get(key); - if(tile) { - // validate the cache - me.cache_validator.getTimestamp(req.params.dbname, req.params.table, function(err, t) { - if(t !== null && tile.timestamp < t) { - me.cache_misses++; - me.cache_invalidated++; - callback(null); - } else { - // stats - me.cache_hits++; - var timestamp = new Date().getTime(); - var delta = timestamp - req.windshaft_start; - tile.cache_time = delta/1000.0; - tile.hits++; - res.header('X-Cache-hit', 'true'); - res.header('Last-Modified', new Date(tile.timestamp*1000).toGMTString()); - // check 304 - var modified_since = req.header('if-modified-since'); - if (modified_since) { - modified_since = Date.parse(modified_since); - if(modified_since && modified_since <= timestamp*1000) { - me.cache_not_modified++; - res.statusCode = 304; - res.end(); - return; - } - } - res.send(tile.tile, tile.headers, 200); - } - }); - } else { - me.cache_misses++; - callback(null); - } - } - - me.afterTileRender = function(req, res, tile, headers, callback) { - var timestamp = new Date().getTime(); - var delta = timestamp - req.windshaft_start; - var key = cache_key(req); - me.cache.put(key, { - tile: tile, - headers: headers, - timestamp: timestamp/1000.0, - render_time: delta/1000.0, - hits: 0}); - ttl.start(key); - update_items(me.cache.size || 0); - callback(null, tile, headers); - } - - me.afterStateChange = function(req, data, callback) { - me.cache_validator.setTimestamp(req.params.dbname, req.params.table, new Date().getTime()/1000.0, function(err, t) { - callback(err, data); - }); - } - - me.getStats = function(include_tile_info, sort_by) { - var total = me.cache_hits + me.cache_misses; - var mem = 0; - var blank_tile_count = 0; - var tile_info = [] - me.cache.forEach(function(key, value) { - if(value.tile.length !== undefined) { - mem += value.tile.length; - if(value.tile.length == BLANK_TILE_SIZE) { - blank_tile_count++; - } - } - if(include_tile_info) { - tile_info.push({ - key: key, - length: value.tile.length, - hits: value.hits, - render_time: value.render_time, - cache_time: value.cache_time - }); - } - }); - sort_by = sort_by || 'hits'; - tile_info.sort(function(a, b) { - return b[sort_by] - a[sort_by]; - }); - return { - cache_hits: me.cache_hits, - cache_misses: me.cache_misses, - cache_invalidated: me.cache_invalidated, - cache_not_modified: me.cache_not_modified, - current_items: me.current_items, - expired: me.expired, - max_items: me.max_items, - memory: mem, - memory_per_item: total ? mem/total: 0, - ratio: total ? me.cache_hits/total: 0, - blank_tile_count: blank_tile_count, - blank_tile_size: blank_tile_count*BLANK_TILE_SIZE, - blank_items_ratio: total? blank_tile_count/total: 0, - tiles: tile_info - }; - } - - return me; -} diff --git a/lib/cartodb/ttl.js b/lib/cartodb/ttl.js deleted file mode 100644 index 443911ba..00000000 --- a/lib/cartodb/ttl.js +++ /dev/null @@ -1,47 +0,0 @@ - -/* - ========================================================== - manages timeouts - * timeout in seconds - * on_delete will be called when time expired with the key as first param - - usage: - ttl = TTL(function(key) { - console.log("10 seconds expired on " + key"); - }, 10); - ========================================================== -*/ -function TTL(on_delete, timeout) { - - var me = { - timeouts: {}, - on_delete: on_delete, - timeout: timeout*1000 - }; - - me.start = function(key) { - var t = me.timeouts[key]; - if (t) { - clearTimeout(t); - } - me.timeouts[key] = setTimeout( - (function(k) { - return function() { - me.on_delete(k); - }; - })(key), - me.timeout); - } - - me.remove = function(key) { - var t = me.timeouts[key]; - if (t) { - clearTimeout(t); - delete me.timeouts[key]; - } - } - - return me; -} - -module.exports = TTL; diff --git a/test/acceptance/cache.js b/test/acceptance/cache.js deleted file mode 100644 index c37f7557..00000000 --- a/test/acceptance/cache.js +++ /dev/null @@ -1,214 +0,0 @@ - -var assert = require('assert') - , tests = module.exports = {} - , _ = require('underscore') - , querystring = require('querystring') - , fs = require('fs') - , th = require(__dirname + '/../test_helper') - , CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft') - , http = require('http') - , Step = require('step') - , CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator') - -var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); - -var cached_server = new CartodbWindshaft(serverOptions); - -tests["first time a tile is request should not be cached"] = function() { - assert.response(cached_server, { - url: '/tiles/test_table_3/6/31/24.png?geom_type=polygon', - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - - }, function(res) { - assert.ok(res.header('X-Cache-hit') === undefined); - - }); - -} - -/* -tests["second time a tile is request should be cached"] = function() { - - var cached_server2 = new CartodbWindshaft(serverOptions); - var url= '/tiles/test_table_2/6/31/24.png'; - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.ok(res.header('X-Cache-hit') !== undefined); - }); - }); -} -*/ - - -tests["LRU tile should be removed"] = function() { - - var urls = ['/tiles/test_table_2/6/31/24.png', - '/tiles/test_table_2/6/31/25.png', - '/tiles/test_table_2/6/31/26.png', - '/tiles/test_table_2/6/31/27.png']; - - //create another server to not take previos test stats into account - var so = _.clone(serverOptions); - _(so).extend({lru_cache: true, lru_cache_size: 3}); - - var _cached_server = new CartodbWindshaft(so); - - function makeReq(url, callback) { - assert.response(_cached_server, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, callback); - } - - Step( - function() { - makeReq(urls[0], this); - }, - function() { - makeReq(urls[1], this); - }, - function() { - makeReq(urls[2], this); - }, - function() { - makeReq(urls[3], this); - }, function() { - assert.response(_cached_server, { - url: urls[0], - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - - }, function(res) { - assert.ok(res.header('X-Cache-hit') === undefined); - var st = so.cacheStats() - assert.eql(st.cache_hits, 0); - assert.eql(st.cache_misses, 5); - assert.eql(st.current_items, 3); - }); - } - ) - -} - -tests["cache should be invalidated"] = function() { - - var url = '/tiles/test_table_2/6/29/27.png'; - var cached_server2 = new CartodbWindshaft(serverOptions); - var cache = CacheValidator(global.environment.redis); - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - cache.setTimestamp('cartodb_test_user_1_db', 'test_table_2', (new Date().getTime()/1000.0)+100, function() { - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(r) { - assert.ok(r.header('X-Cache-hit') === undefined); - }); - }); - }); - -}; - -tests["Last-Modified header should be sent"] = function() { - var cached_server2 = new CartodbWindshaft(serverOptions); - var url= '/tiles/test_table/6/31/22.png'; - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.ok(res.header('X-Cache-hit') !== undefined); - var last_modified = res.header('Last-Modified'); - assert.ok(last_modified !== undefined); - assert.response(cached_server2, { - url: url, - headers: { - host: 'vizzuality.localhost.lan', - 'if-modified-since': last_modified - }, - method: 'GET' - }, { - status: 304 - }); - }); - }); -} - -tests["TTL should invalidate a tile"] = function() { - - var url = '/tiles/test_table_2/6/31/24.png'; - - //create another server to not take previos test stats into account - var so = _.clone(serverOptions); - _(so).extend({ttl_timeout: 1}); - - var _cached_server = new CartodbWindshaft(so); - // cache it - assert.response(_cached_server, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - console.log("WAIT A LITTLE BIT PLEASE"); - - // test before invalidating - setTimeout(function() { - var st = so.cacheStats(); - assert.eql(st.expired, 0); - }, 500); - - // test after invalidation - setTimeout(function() { - assert.response(_cached_server, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.ok(res.header('X-Cache-hit') === undefined); - var st = so.cacheStats(); - assert.eql(st.expired, 1); - }); - }, 2000); - }); -} diff --git a/test/acceptance/cache_validator.js b/test/acceptance/cache_validator.js deleted file mode 100644 index 55bd714e..00000000 --- a/test/acceptance/cache_validator.js +++ /dev/null @@ -1,24 +0,0 @@ - - -var assert = require('assert'), - tests = module.exports = {}, - th = require(__dirname + '/../test_helper'), - CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator') - -var cache = CacheValidator(global.environment.redis); - -tests["should get the timestamp it sets"] = function() { - var d = new Date().getTime(); - cache.setTimestamp('mydatabase', 'mytable', d, function() { - cache.getTimestamp('mydatabase', 'mytable', function(err, t) { - assert.eql(t, d); - }) - }); -} - -tests["should get null when timestamp is not set"] = function() { - var d = new Date().getTime(); - cache.getTimestamp('mydatabase', 'mytable2', function(err, t) { - assert.ok(t === null); - }) -} From 84744253756016df524bb28ad72e3afd02d609b9 Mon Sep 17 00:00:00 2001 From: javi santana Date: Wed, 30 Nov 2011 19:59:28 +0100 Subject: [PATCH 4/7] added varnish invalidation code --- app.js | 1 - config/environments/development.js | 6 +- config/environments/production.js | 5 +- config/environments/test.js | 5 +- lib/cartodb/cache_validator.js | 85 +++------------ lib/cartodb/cartodb_windshaft.js | 38 +++---- lib/cartodb/server_options.js | 6 +- lib/cartodb/varnish.js | 166 +++++++++++++++++++++++++++++ 8 files changed, 204 insertions(+), 108 deletions(-) create mode 100644 lib/cartodb/varnish.js diff --git a/app.js b/app.js index c8e10ad8..7a56422f 100755 --- a/app.js +++ b/app.js @@ -30,7 +30,6 @@ _.extend(global.settings, global.environment); var Windshaft = require('windshaft'); var serverOptions = require('./lib/cartodb/server_options'); -var Cache = require('./lib/cartodb/tile_cache'); ws = CartodbWindshaft(serverOptions); ws.listen(global.environment.windshaft_port); diff --git a/config/environments/development.js b/config/environments/development.js index ed8674cb..4cf40a8c 100644 --- a/config/environments/development.js +++ b/config/environments/development.js @@ -5,7 +5,7 @@ module.exports.redis = {host: '127.0.0.1', idleTimeoutMillis: 1, reapIntervalMillis: 1}; module.exports.windshaft_port = 8181; -module.exports.lru_cache = false; -module.exports.lru_cache_size = 10000; module.exports.enable_cors = true; -module.exports.ttl_timeout = 5; +module.exports.varnish_host = 'localhost'; +module.exports.varnish_port = 6082; +module.exports.cache_enabled = true; diff --git a/config/environments/production.js b/config/environments/production.js index 5d4bdaad..6307840e 100644 --- a/config/environments/production.js +++ b/config/environments/production.js @@ -2,6 +2,7 @@ module.exports.name = 'production'; module.exports.postgres = {user: 'tileuser', host: '127.0.0.1', port: 6432, max_size: 4}; module.exports.redis = {host: '127.0.0.1', port: 6379}; module.exports.windshaft_port = 8181; -module.exports.lru_cache = false; -module.exports.lru_cache_size = 10000; module.exports.ttl_timeout = 600; // 10 minutes +module.exports.varnish_host = 'localhost'; +module.exports.varnish_port = 6082 +module.exports.cache_enabled = true; diff --git a/config/environments/test.js b/config/environments/test.js index 86b9e524..6d912d0c 100644 --- a/config/environments/test.js +++ b/config/environments/test.js @@ -5,6 +5,7 @@ module.exports.redis = {host: '127.0.0.1', idleTimeoutMillis: 1, reapIntervalMillis: 1}; module.exports.windshaft_port = 8080; -module.exports.lru_cache = true; -module.exports.lru_cache_size = 10000; module.exports.enable_cors = true; +module.exports.varnish_host = ''; +module.exports.varnish_port = null; +module.exports.cache_enabled = false; diff --git a/lib/cartodb/cache_validator.js b/lib/cartodb/cache_validator.js index 29d53e78..803f9238 100644 --- a/lib/cartodb/cache_validator.js +++ b/lib/cartodb/cache_validator.js @@ -1,76 +1,17 @@ -// -// this module allows to invalidate cache for a table or check if the cache for a table -// is valid -// -// usage: -// -// Cache.getTimestamp('table_name', function(err, timestamp) { -// }); -// -// Cache.setTimestamp('table_name', timestamp, function(err, result) { -// }); -// -// - var _ = require('underscore'), - RedisPool = require('./redis_pool'), - Step = require('step') + Varnish = require('./varnish') +var varnish_queue = null; -function Cache(redis_opts) { - - var redis_pool = new RedisPool(redis_opts); - - var me = {} - - var redis_options = { - db: 0 - } - - var redisCommand = function(fn, callback) { - var redis_client; - Step( - function getRedisClient(){ - redis_pool.acquire(redis_options.db, this); - }, - function getDataForTable(err, redis) { - if (err) throw err; - redis_client = redis; - fn(redis_client, this); - }, - function exit(err, data) { - if (!_.isUndefined(redis_client)) - redis_pool.release(redis_options.db, redis_client); - if (callback) { - callback(err, data); - } - } - ); - } - - // get the timestamp for the table - me.getTimestamp = function(database, table, callback) { - redisCommand(function(redis, step) { - redis.GET("cache:" + database + ":" + table + ":last_updated_at", step); - }, function(err, t) { - if(t != null) - callback(err, parseFloat(t)); - else - callback(err, t); - }); - } - - me.setTimestamp = function(database, table, timestamp, callback) { - timestamp = timestamp || new Date().getTime(); - redisCommand(function(redis, step) { - redis.SET("cache:" + database + ":" + table + ":last_updated_at", timestamp, step); - }, callback); - } - - return me; -}; - - -module.exports = function(redis_opts) { - return new Cache(redis_opts); +function init(host, port) { + varnish_queue = new Varnish.VarnishQueue(host, port); +} + +function invalidate_db(dbname) { + varnish_queue.run_cmd('purge obj.http.X-Cache-Channel == ' + dbname); +} + +module.exports = { + init: init, + invalidate_db: invalidate_db } diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 568fc002..a28e976e 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -2,29 +2,27 @@ var _ = require('underscore') , Step = require('step') , Windshaft = require('windshaft') - , Cache = require('./tile_cache'); + , Cache = require('./cache_validator') var CartodbWindshaft = function(serverOptions) { - // set cache if requested - if(serverOptions.lru_cache) { - var lru_cache = Cache.LRUcache(serverOptions.lru_cache_size || 10000, - serverOptions.redis, - serverOptions.ttl_timeout); - _.extend(serverOptions, { - beforeTileRender: lru_cache.beforeTileRender, - afterTileRender: lru_cache.afterTileRender, - cacheStats: lru_cache.getStats, - afterStateChange: lru_cache.afterStateChange - }); - } - // set the cache chanel info to invalidate the cache on the frontend server serverOptions.afterTileRender = function(req, res, tile, headers, callback) { res.header('X-Cache-Channel', req.params.dbname); callback(null, tile, headers); }; + if(serverOptions.cache_enabled) { + console.log("cache invalidation enabled, varnish on " serverOptions.varnish_host, ' ', serverOptions.varnish_port); + Cache.init(serverOptions.varnish_host, serverOptions.varnish_port); + serverOptions.afterStateChange = function(req, data, callback) { + Cache.invalidate_db(req.params.dbname); + callback(null, data); + } + } + + + // boot var ws = new Windshaft.Server(serverOptions); @@ -46,6 +44,7 @@ var CartodbWindshaft = function(serverOptions) { ); }); + /** * Helper to allow access to metadata to be used in embedded maps. */ @@ -64,17 +63,6 @@ var CartodbWindshaft = function(serverOptions) { ); }); - /** - * tile cache stats - */ - ws.get('/cache', function(req, res){ - if(serverOptions.cacheStats) { - res.send(serverOptions.cacheStats(req.query.tile_info, req.query.sort_by)); - } else { - res.send("Cache no enabled") - } - }); - return ws; } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 96e37ee9..e37c2c5e 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -8,9 +8,9 @@ module.exports = function(){ grainstore: {datasource: global.environment.postgres}, redis: global.environment.redis, enable_cors: global.environment.enable_cors, - lru_cache: global.environment.lru_cache, - lru_cache_size: global.environment.lru_cache_size, - ttl_timeout: global.environment.ttl_timeout + varnish_host: global.environment.varnish_host, + varnish_port: global.environment.varnish_port, + cache_enabled: global.environment.cache_enabled }; /** diff --git a/lib/cartodb/varnish.js b/lib/cartodb/varnish.js new file mode 100644 index 00000000..3360a0a9 --- /dev/null +++ b/lib/cartodb/varnish.js @@ -0,0 +1,166 @@ + +/** + * this module implements varnish telnet management protocol + * https://www.varnish-cache.org/trac/wiki/ManagementPort + */ + +var net = require('net') + +function VarnishClient(host, port, ready_callback) { + + var ready = false; + var cmd_callback = null; + var client = null; + + function connect() { + ready = false; + client = net.createConnection(port, host); + } + + connect(); + + client.on('data', function(data) { + data = data.toString(); + lines = data.split('\n', 2); + if(lines.length == 2) { + var tk = lines[0].split(' ') + var code = parseInt(tk[0], 10); + var body_length = parseInt(tk[1], 10); + var body = lines[1]; + if(!ready) { + ready = true; + ready_callback(); + } else if(cmd_callback) { + var c = cmd_callback + cmd_callback = null; + c(null, code, body); + } + } + + }); + + // sends the command to the server + function _send(cmd, callback) { + cmd_callback = callback; + client.write(cmd + '\n'); + } + + // run command if there is no peding response + // fist param of the callback are the error, null + // if all went ok + this.run_cmd = function(cmd, callback) { + if(!cmd_callback) { + _send(cmd, callback); + } else { + callback('response pending'); + } + } + + // close the connection + this.close = function() { + client.end(); + ready = false; + } + +} + + +function VarnishPool(opts, ready) { + var resources = []; + var available = []; + + for(var i = 0; i < opts.pool_size; ++i) { + var v = new VarnishClient(opts.host, opts.port, function() { + resources.push(v); + available.push(v); + }); + } + + this.run_cmd = function(cmd, callback) { + var v = available.pop() + if(v) { + v.run_cmd(cmd, function(err, status_code, body) { + callback(err, status_code, body); + available.push(v); + ready(); + }); + } else { + callback('no clients available'); + } + } + + this.close = function() { + for(var i = 0; i < resources.length; ++i) { + resources[i].close(); + } + } +} + +function VarnishQueue(host, port) { + + var queue = []; + var ready = false; + + var client = new VarnishClient(host, port, function(err) { + ready = true; + _send_pending(); + }); + + function _send_pending(empty_callback) { + var c = queue.pop(); + if(!c) return; + client.run_cmd(c, function() { + if(queue.length > 0) { + process.nextTick(_send_pending); + } else { + if(empty_callback) { + empty_callback(); + } + } + }); + } + + this.run_cmd = function(cmd) { + queue.push(cmd); + if(ready) { + _send_pending(); + } + } + + this.end = function() { + _send_pending(function() { + client.close(); + }); + } + +} + +/* +var queue = new VarnishQueue('localhost', 6082) +setInterval(function() { + queue.run_cmd('purge obj.http.url == /') +}, 10) +*/ +/* +v = new VarnishClient('localhost', 6082, function(err) { + console.log('connected'); + v.run_cmd('purge obj.http.url == /', function(err, code, body) { + console.log(code); + console.log(body); + v.close(); + }); +}); + +pool = new VarnishPool({ + host: 'locahost', + port: 6082, + pool_size: 5 +}); +/* +v.close(); +*/ + +module.exports = { + VarnishClient: VarnishClient, + VarnishQueue: VarnishQueue +} From f4f7f6300d2266bfc958fd2f1ea90d936c2b29e8 Mon Sep 17 00:00:00 2001 From: javi santana Date: Wed, 30 Nov 2011 20:00:41 +0100 Subject: [PATCH 5/7] fixed log --- lib/cartodb/cartodb_windshaft.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index a28e976e..26e84eed 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -13,7 +13,7 @@ var CartodbWindshaft = function(serverOptions) { }; if(serverOptions.cache_enabled) { - console.log("cache invalidation enabled, varnish on " serverOptions.varnish_host, ' ', serverOptions.varnish_port); + console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); Cache.init(serverOptions.varnish_host, serverOptions.varnish_port); serverOptions.afterStateChange = function(req, data, callback) { Cache.invalidate_db(req.params.dbname); From b86d3ae35e455f09fb7d150b9e81fe77db146865 Mon Sep 17 00:00:00 2001 From: javi Date: Thu, 1 Dec 2011 10:32:27 +0100 Subject: [PATCH 6/7] added test for cache validator --- test/acceptance/cache_validator.js | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/acceptance/cache_validator.js diff --git a/test/acceptance/cache_validator.js b/test/acceptance/cache_validator.js new file mode 100644 index 00000000..6e39dd8a --- /dev/null +++ b/test/acceptance/cache_validator.js @@ -0,0 +1,35 @@ +var assert = require('assert'); +var net = require('net'); +require(__dirname + '/../test_helper'); +var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator'); +var tests = module.exports = {}; + +function VarnishEmu(on_cmd_recieved) { + var self = this; + var welcome_msg = 'hi, im a varnish emu, right?'; + + self.commands_recieved = []; + + var server = net.createServer(function (socket) { + var command = ''; + socket.write("200 " + welcome_msg.length + "\n"); + socket.write(welcome_msg); + socket.on('data', function(data) { + self.commands_recieved.push(data); + on_cmd_recieved && on_cmd_recieved(self.commands_recieved); + socket.write('200 0\n'); + }); + }); + server.listen(1337, "127.0.0.1"); +} + +tests['working'] = function() { assert.ok(true); }; + +tests['should call purge on varnish when invalidate database'] = function() { + var varnish = new VarnishEmu(function(cmds) { + assert.ok(cmds.length == 1); + assert.equal('purge obj.http.X-Cache-Channel == test_cache\n', cmds[0]); + }); + CacheValidator.init('localhost', 1337); + CacheValidator.invalidate_db('test_cache'); +} From db55c08be1773b65d6b7a1f04e05d86e1afe29ce Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 5 Dec 2011 15:50:47 +0100 Subject: [PATCH 7/7] some varnish sanity checks --- lib/cartodb/varnish.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/cartodb/varnish.js b/lib/cartodb/varnish.js index 3360a0a9..977f5bdc 100644 --- a/lib/cartodb/varnish.js +++ b/lib/cartodb/varnish.js @@ -8,6 +8,7 @@ var net = require('net') function VarnishClient(host, port, ready_callback) { + var self = this; var ready = false; var cmd_callback = null; var client = null; @@ -39,6 +40,14 @@ function VarnishClient(host, port, ready_callback) { }); + client.on('error', function(err) { + console.log("[ERROR] some problem in varnish connection"); + }); + + client.on('close', function() { + self.close(); + }); + // sends the command to the server function _send(cmd, callback) { cmd_callback = callback; @@ -98,6 +107,7 @@ function VarnishPool(opts, ready) { function VarnishQueue(host, port) { + var MAX_QUEUE = 2000; var queue = []; var ready = false; @@ -122,6 +132,10 @@ function VarnishQueue(host, port) { this.run_cmd = function(cmd) { queue.push(cmd); + if(queue.length > MAX_QUEUE) { + console.log("varnish commando queu too long, removing commands"); + queue.pop(); + } if(ready) { _send_pending(); }