cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'standalone-server' into standalone-server-mvt

Browse Source 
Conflicts:
	npm-shrinkwrap.json
This commit is contained in:
Raul Ochoa 2015-09-08 15:46:23 +02:00
commit b2e1e5361f
37 changed files with 2169 additions and 1117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.jshintrc
View File

@ -7,8 +7,8 @@
// // Enforcing
// "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
// "camelcase" : false, // true: Identifiers must be in camelCase
// "curly" : true, // true: Require {} for every new block or scope
// "eqeqeq" : true, // true: Require triple equals (===) for comparison
"curly" : true, // true: Require {} for every new block or scope
"eqeqeq" : true, // true: Require triple equals (===) for comparison
"forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty()
"freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc.
"immed" : true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
@ -31,7 +31,7 @@
// "maxparams" : false, // {int} Max number of formal params allowed per function
// "maxdepth" : false, // {int} Max depth of nested blocks (within functions)
// "maxstatements" : false, // {int} Max number statements per function
"maxcomplexity" : 8, // {int} Max cyclomatic complexity per function
"maxcomplexity" : 6, // {int} Max cyclomatic complexity per function
"maxlen" : 120, // {int} Max number of characters per line
//
// // Relaxing

12
Makefile
View File

@ -25,13 +25,9 @@ test: config/environments/test.js
test/unit/cartodb/cache/model/*.js \
test/integration/*.js \
test/acceptance/*.js \
test/acceptance/cache/*.js
test-ported: config/environments/test.js
@echo "***tests ported***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \
test/unit/cartodb/ported/*.js \
test/acceptance/ported/*.js
test/acceptance/cache/*.js \
test/acceptance/ported/*.js \
test/unit/cartodb/ported/*.js
test-unit: config/environments/test.js
@echo "***tests***"
@ -54,7 +50,7 @@ jshint:
@echo "***jshint***"
@./node_modules/.bin/jshint lib/ test/ app.js
test-all: jshint test test-ported
test-all: jshint test
coverage:
@RUNTESTFLAGS=--with-coverage make test

58
NEWS.md
View File

@ -1,8 +1,64 @@
# Changelog
## 2.12.1
Released 2015-mm-dd
## 2.12.0
Released 2015-08-27
Announcements:
- Upgrades windshaft to [0.51.0](https://github.com/CartoDB/Windshaft/releases/tag/0.51.0)
New features:
- Make http and https globalAgent options configurable
* If config is not provided it configures them with default values
## 2.11.0
Released 2015-08-26
Announcements:
- Upgrades windshaft to [0.50.0](https://github.com/CartoDB/Windshaft/releases/tag/0.50.0)
## 2.10.0
Released 2015-08-18
New features:
- Exposes metatile cache configuration for tilelive-mapnik, see configuration sample files for more information.
Announcements:
- Upgrades windshaft to [0.49.0](https://github.com/CartoDB/Windshaft/releases/tag/0.49.0)
## 2.9.0
Released 2015-08-06
New features:
- Send memory usage stats
## 2.8.0
Released 2015-07-15
Announcements:
- Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0)
## 2.7.2
Released 2015-mm-dd
Released 2015-07-14
Enhancements:
- Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names
## 2.7.1

30
app.js
View File

@ -1,6 +1,10 @@
var http = require('http');
var https = require('https');
var path = require('path');
var fs = require('fs');
var _ = require('underscore');
var ENVIRONMENT;
if ( process.argv[2] ) {
ENVIRONMENT = process.argv[2];
@ -38,6 +42,17 @@ if (global.environment.uv_threadpool_size) {
process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
}
// set global HTTP and HTTPS agent default configurations
// ref https://nodejs.org/api/http.html#http_new_agent_options
var agentOptions = _.defaults(global.environment.httpAgent || {}, {
keepAlive: false,
keepAliveMsecs: 1000,
maxSockets: Infinity,
maxFreeSockets: 256
});
http.globalAgent = new http.Agent(agentOptions);
https.globalAgent = new https.Agent(agentOptions);
if ( global.environment.log_filename ) {
var logdir = path.dirname(global.environment.log_filename);
// See cwd inlog4js.configure call below
@ -79,12 +94,19 @@ server.listen(serverOptions.bind.port, serverOptions.bind.host);
var version = require("./package").version;
server.on('listening', function() {
console.log(
"Windshaft tileserver %s started on %s:%s (%s)",
version, serverOptions.bind.host, serverOptions.bind.port, ENVIRONMENT
);
console.log(
"Windshaft tileserver %s started on %s:%s PID=%d (%s)",
version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
);
});
setInterval(function() {
var memoryUsage = process.memoryUsage();
Object.keys(memoryUsage).forEach(function(k) {
global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
});
}, 5000);
process.on('SIGHUP', function() {
global.log4js.clearAndShutdownAppenders(function() {
global.log4js.configure(log4js_config);

37
config/environments/development.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'development'
,port: 8181
,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
@ -86,8 +89,10 @@ var config = {
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
@ -96,6 +101,17 @@ var config = {
// wasted time.
metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format
formatMetatile: {
png: 2,
@ -146,6 +162,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
}
,millstone: {
@ -180,6 +206,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
}
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: {
host: 'localhost',
port: 6082, // the por for the telnet interface where varnish is listening to

37
config/environments/production.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'production'
,port: 8181
,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
@ -80,8 +83,10 @@ var config = {
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
@ -90,6 +95,17 @@ var config = {
// wasted time.
metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format
formatMetatile: {
png: 2,
@ -140,6 +156,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
}
,millstone: {
@ -174,6 +200,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
}
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: {
host: 'localhost',
port: 6082, // the por for the telnet interface where varnish is listening to

37
config/environments/staging.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'production'
,port: 8181
,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
@ -80,8 +83,10 @@ var config = {
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
@ -90,6 +95,17 @@ var config = {
// wasted time.
metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format
formatMetatile: {
png: 2,
@ -140,6 +156,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
}
,millstone: {
@ -174,6 +200,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
}
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: {
host: 'localhost',
port: 6082, // the por for the telnet interface where varnish is listening to

37
config/environments/test.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'test'
,port: 8888
,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
@ -80,8 +83,10 @@ var config = {
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: {
// The size of the pool of internal mapnik renderers
// Check the configuration of uv_threadpool_size to use suitable value
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// Metatile is the number of tiles-per-side that are going
@ -90,6 +95,17 @@ var config = {
// wasted time.
metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format
formatMetatile: {
png: 2,
@ -142,6 +158,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
}
,millstone: {
@ -176,6 +202,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
}
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: {
host: '',
port: null, // the por for the telnet interface where varnish is listening to

141
lib/cartodb/api/auth_api.js Normal file
View File

@ -0,0 +1,141 @@
var assert = require('assert');
var step = require('step');
/**
*
* @param {PgConnection} pgConnection
* @param metadataBackend
* @param {MapStore} mapStore
* @param {TemplateMaps} templateMaps
* @constructor
* @type {AuthApi}
*/
function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.mapStore = mapStore;
this.templateMaps = templateMaps;
}
module.exports = AuthApi;
// Check if a request is authorized by a signer
//
// @param req express request object
// @param callback function(err, signed_by) signed_by will be
// null if the request is not signed by anyone
// or will be a string cartodb username otherwise.
//
AuthApi.prototype.authorizedBySigner = function(req, callback) {
if ( ! req.params.token || ! req.params.signer ) {
return callback(null, false); // no signer requested
}
var self = this;
var layergroup_id = req.params.token;
var auth_token = req.params.auth_token;
this.mapStore.load(layergroup_id, function(err, mapConfig) {
if (err) {
return callback(err);
}
var authorized = self.templateMaps.isAuthorized(mapConfig.obj().template, auth_token);
return callback(null, authorized);
});
};
// Check if a request is authorized by api_key
//
// @param user
// @param req express request object
// @param callback function(err, authorized)
// NOTE: authorized is expected to be 0 or 1 (integer)
//
AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) {
var givenKey = req.query.api_key || req.query.map_key;
if ( ! givenKey && req.body ) {
// check also in request body
givenKey = req.body.api_key || req.body.map_key;
}
if ( ! givenKey ) {
return callback(null, 0); // no api key, no authorization...
}
var self = this;
step(
function () {
self.metadataBackend.getUserMapKey(user, this);
},
function checkApiKey(err, val){
assert.ifError(err);
return val && givenKey === val;
},
function finish(err, authorized) {
callback(err, authorized);
}
);
};
/**
* Check access authorization
*
* @param req - standard req object. Importantly contains table and host information
* @param callback function(err, allowed) is access allowed not?
*/
AuthApi.prototype.authorize = function(req, callback) {
var self = this;
var user = req.context.user;
step(
function () {
self.authorizedByAPIKey(user, req, this);
},
function checkApiKey(err, authorized){
if (req.profiler) {
req.profiler.done('authorizedByAPIKey');
}
assert.ifError(err);
// if not authorized by api_key, continue
if (!authorized) {
// not authorized by api_key, check if authorized by signer
return self.authorizedBySigner(req, this);
}
// authorized by api key, login as the given username and stop
self.pgConnection.setDBAuth(user, req.params, function(err) {
callback(err, true); // authorized (or error)
});
},
function checkSignAuthorized(err, authorized) {
if (err) {
return callback(err);
}
if ( ! authorized ) {
// request not authorized by signer.
// if no signer name was given, let dbparams and
// PostgreSQL do the rest.
//
if ( ! req.params.signer ) {
return callback(null, true); // authorized so far
}
// if signer name was given, return no authorization
return callback(null, false);
}
self.pgConnection.setDBAuth(user, req.params, function(err) {
if (req.profiler) {
req.profiler.done('setDBAuth');
}
callback(err, true); // authorized (or error)
});
}
);
};

45
lib/cartodb/api/query_tables_api.js
View File

@ -14,7 +14,7 @@ module.exports = QueryTablesApi;
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback);
};
@ -25,9 +25,9 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
// This is an Array, so no need to split into parts
var tableNames = rows[0].cdb_querytablestext;
callback(null, tableNames);
}
@ -35,7 +35,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
var query = [
'WITH querytables AS (',
'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
')',
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m',
@ -48,14 +48,14 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
if (err || rows.length === 0) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables and last updated time: ' + msg));
callback(new Error('could not fetch affected tables or last updated time: ' + msg));
return;
}
var result = rows[0];
var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
// This is an Array, so no need to split into parts
var tableNames = result.tablenames;
var lastUpdatedTime = result.max || 0;
@ -65,6 +65,35 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
});
}
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, 0);
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
this.pgQueryRunner.run(username, query, handleLastUpdatedTimeRows, callback);
};
function handleLastUpdatedTimeRows(err, rows, callback) {
if (err) {
var msg = err.message ? err.message : err;
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var lastUpdated = 0;
if (rows.length !== 0) {
lastUpdated = rows[0].max || 0;
}
return callback(null, lastUpdated*1000);
}
function prepareSql(sql) {
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')

17
lib/cartodb/backends/pg_connection.js
View File

@ -1,3 +1,4 @@
var assert = require('assert');
var step = require('step');
var _ = require('underscore');
@ -29,19 +30,21 @@ PgConnection.prototype.setDBAuth = function(username, params, callback) {
self.metadataBackend.getUserId(username, this);
},
function(err, user_id) {
if (err) throw err;
assert.ifError(err);
user_params.user_id = user_id;
var dbuser = _.template(auth_user, user_params);
_.extend(params, {dbuser:dbuser});
// skip looking up user_password if postgres_auth_pass
// doesn't contain the "user_password" label
if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) {
return null;
}
self.metadataBackend.getUserDBPass(username, this);
},
function(err, user_password) {
if (err) throw err;
assert.ifError(err);
user_params.user_password = user_password;
if ( auth_pass ) {
var dbpass = _.template(auth_pass, user_params);
@ -81,12 +84,14 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
self.metadataBackend.getUserDBConnectionParams(dbowner, this);
},
function extendParams(err, dbParams){
if (err) throw err;
assert.ifError(err);
// we don't want null values or overwrite a non public user
if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
if (params.dbuser !== 'publicuser' || !dbParams.dbuser) {
delete dbParams.dbuser;
}
if ( dbParams ) _.extend(params, dbParams);
if ( dbParams ) {
_.extend(params, dbParams);
}
return null;
},
function finish(err) {

9
lib/cartodb/backends/pg_query_runner.js
View File

@ -1,3 +1,4 @@
var assert = require('assert');
var PSQL = require('cartodb-psql');
var step = require('step');
@ -18,15 +19,11 @@ PgQueryRunner.prototype.run = function(username, query, queryHandler, callback)
self.pgConnection.setDBAuth(username, params, this);
},
function setConn(err) {
if (err) {
throw err;
}
assert.ifError(err);
self.pgConnection.setDBConn(username, params, this);
},
function executeQuery(err) {
if (err) {
throw err;
}
assert.ifError(err);
var psql = new PSQL({
user: params.dbuser,
pass: params.dbpass,

51
lib/cartodb/backends/template_maps.js
View File

@ -1,3 +1,4 @@
var assert = require('assert');
var crypto = require('crypto');
var step = require('step');
var _ = require('underscore');
@ -21,7 +22,9 @@ var util = require('util');
//
//
function TemplateMaps(redis_pool, opts) {
if (!(this instanceof TemplateMaps)) return new TemplateMaps();
if (!(this instanceof TemplateMaps)) {
return new TemplateMaps();
}
EventEmitter.call(this);
@ -76,13 +79,15 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
that.redis_pool.acquire(db, this);
},
function executeQuery(err, data) {
if ( err ) throw err;
assert.ifError(err);
redisClient = data;
redisArgs.push(this);
redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
},
function releaseRedisClient(err, data) {
if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
if ( ! _.isUndefined(redisClient) ) {
that.redis_pool.release(db, redisClient);
}
callback(err, data);
}
);
@ -92,7 +97,7 @@ var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
// jshint maxcomplexity:15
o._checkInvalidTemplate = function(template) {
if ( template.version != '0.0.1' ) {
if ( template.version !== '0.0.1' ) {
return new Error("Unsupported template version " + template.version);
}
var tplname = template.name;
@ -131,10 +136,12 @@ o._checkInvalidTemplate = function(template) {
case 'open':
break;
case 'token':
if ( ! _.isArray(auth.valid_tokens) )
if ( ! _.isArray(auth.valid_tokens) ) {
return new Error("Invalid 'token' authentication: missing valid_tokens");
if ( ! auth.valid_tokens.length )
}
if ( ! auth.valid_tokens.length ) {
return new Error("Invalid 'token' authentication: no valid_tokens");
}
break;
default:
return new Error("Unsupported authentication method: " + auth.method);
@ -214,9 +221,7 @@ o.addTemplate = function(owner, template, callback) {
self._redisCmd('HLEN', [ userTemplatesKey ], this);
},
function installTemplateIfDoesNotExist(err, numberOfTemplates) {
if ( err ) {
throw err;
}
assert.ifError(err);
if ( limit && numberOfTemplates >= limit ) {
throw new Error("User '" + owner + "' reached limit on number of templates " +
"("+ numberOfTemplates + "/" + limit + ")");
@ -224,9 +229,7 @@ o.addTemplate = function(owner, template, callback) {
self._redisCmd('HSETNX', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
},
function validateInstallation(err, wasSet) {
if ( err ) {
throw err;
}
assert.ifError(err);
if ( ! wasSet ) {
throw new Error("Template '" + templateName + "' of user '" + owner + "' already exists");
}
@ -259,9 +262,7 @@ o.delTemplate = function(owner, tpl_id, callback) {
self._redisCmd('HDEL', [ self.key_usr_tpl({ owner:owner }), tpl_id ], this);
},
function handleDeletion(err, deleted) {
if (err) {
throw err;
}
assert.ifError(err);
if (!deleted) {
throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
}
@ -306,7 +307,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
var templateName = template.name;
if ( tpl_id != templateName ) {
if ( tpl_id !== templateName ) {
return callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + templateName + "')"));
}
@ -317,18 +318,14 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
self._redisCmd('HGET', [ userTemplatesKey, tpl_id ], this);
},
function updateTemplate(err, currentTemplate) {
if (err) {
throw err;
}
assert.ifError(err);
if (!currentTemplate) {
throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
}
self._redisCmd('HSET', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
},
function handleTemplateUpdate(err, didSetNewField) {
if (err) {
throw err;
}
assert.ifError(err);
if (didSetNewField) {
console.warn('New template created on update operation');
}
@ -372,7 +369,7 @@ o.getTemplate = function(owner, tpl_id, callback) {
self._redisCmd('HGET', [ self.key_usr_tpl({owner:owner}), tpl_id ], this);
},
function parseTemplate(err, tpl_val) {
if ( err ) throw err;
assert.ifError(err);
return JSON.parse(tpl_val);
},
function finish(err, tpl) {
@ -472,8 +469,12 @@ o.instance = function(template, params) {
var layergroup = JSON.parse(JSON.stringify(template.layergroup));
for (var i=0; i<layergroup.layers.length; ++i) {
var lyropt = layergroup.layers[i].options;
if ( lyropt.cartocss ) lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
if ( lyropt.sql) lyropt.sql = _replaceVars(lyropt.sql, all_params);
if ( lyropt.cartocss ) {
lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
}
if ( lyropt.sql) {
lyropt.sql = _replaceVars(lyropt.sql, all_params);
}
// Anything else ?
}

24
lib/cartodb/cache/layergroup_affected_tables.js vendored Normal file
View File

@ -0,0 +1,24 @@
var LruCache = require('lru-cache');
function LayergroupAffectedTables() {
// dbname + layergroupId -> affected tables cache
this.cache = new LruCache({ max: 2000 });
}
module.exports = LayergroupAffectedTables;
LayergroupAffectedTables.prototype.hasAffectedTables = function(dbName, layergroupId) {
return this.cache.has(createKey(dbName, layergroupId));
};
LayergroupAffectedTables.prototype.set = function(dbName, layergroupId, affectedTables) {
this.cache.set(createKey(dbName, layergroupId), affectedTables);
};
LayergroupAffectedTables.prototype.get = function(dbName, layergroupId) {
return this.cache.get(createKey(dbName, layergroupId));
};
function createKey(dbName, layergroupId) {
return dbName + ':' + layergroupId;
}

24
lib/cartodb/cache/model/database_tables_entry.js vendored Normal file
View File

@ -0,0 +1,24 @@
var crypto = require('crypto');
function DatabaseTables(dbName, tableNames) {
this.namespace = 't';
this.dbName = dbName;
this.tableNames = tableNames;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tableNames.map(function(tableName) {
return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
}.bind(this));
};
DatabaseTables.prototype.getCacheChannel = function() {
return this.dbName + ':' + this.tableNames.join(',');
};
function shortHashKey(target) {
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
}

62
lib/cartodb/cache/named_map_provider_cache.js vendored Normal file
View File

@ -0,0 +1,62 @@
var _ = require('underscore');
var dot = require('dot');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
var templateName = require('../backends/template_maps').templateName;
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.providerCache = new LruCache({ max: 2000 });
}
module.exports = NamedMapProviderCache;
NamedMapProviderCache.prototype.get = function(user, templateId, config, authToken, params) {
var namedMapKey = createNamedMapKey(user, templateId);
var namedMapProviders = this.providerCache.get(namedMapKey) || {};
var providerKey = createProviderKey(config, authToken, params);
if (!namedMapProviders.hasOwnProperty(providerKey)) {
namedMapProviders[providerKey] = new NamedMapMapConfigProvider(
this.templateMaps,
this.pgConnection,
this.userLimitsApi,
this.queryTablesApi,
user,
templateId,
config,
authToken,
params
);
this.providerCache.set(namedMapKey, namedMapProviders);
}
return namedMapProviders[providerKey];
};
NamedMapProviderCache.prototype.invalidate = function(user, templateId) {
this.providerCache.del(createNamedMapKey(user, templateId));
};
function createNamedMapKey(user, templateId) {
return user + ':' + templateName(templateId);
}
var providerKey = '{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}';
var providerKeyTpl = dot.template(providerKey);
function createProviderKey(config, authToken, params) {
var tplValues = _.defaults({}, params, {
authToken: authToken || '',
configHash: NamedMapMapConfigProvider.configHash(config),
layer: '',
format: '',
scale_factor: 1
});
return providerKeyTpl(tplValues);
}

14
lib/cartodb/cache/surrogate_keys_cache.js vendored
View File

@ -16,9 +16,21 @@ module.exports = SurrogateKeysCache;
* @param cacheObject should respond to `key() -> String` method
*/
SurrogateKeysCache.prototype.tag = function(response, cacheObject) {
response.header('Surrogate-Key', cacheObject.key());
var newKey = cacheObject.key();
response.header('Surrogate-Key', appendSurrogateKey(
response.header('Surrogate-Key'),
Array.isArray(newKey) ? cacheObject.key().join(' ') : newKey
));
};
function appendSurrogateKey(currentKey, newKey) {
if (!!currentKey) {
newKey = currentKey + ' ' + newKey;
}
return newKey;
}
/**
* @param cacheObject should respond to `key() -> String` method
* @param {Function} callback

107
lib/cartodb/controllers/layergroup.js
View File

@ -4,6 +4,7 @@ var step = require('step');
var cors = require('../middleware/cors');
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
/**
* @param app
@ -11,16 +12,23 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'
* @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend
* @param {{UserLimitsApi}} userLimitsApi
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor
*/
function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi) {
function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, surrogateKeysCache,
userLimitsApi, queryTablesApi, layergroupAffectedTables) {
this.app = app;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.layergroupAffectedTables = layergroupAffectedTables;
}
module.exports = LayergroupController;
@ -62,7 +70,7 @@ LayergroupController.prototype.attributes = function(req, res) {
var statusCode = self.app.findStatusCode(err);
self.app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err);
} else {
self.app.sendResponse(res, [tile, 200]);
self.sendResponse(req, res, [tile, 200]);
}
}
);
@ -87,8 +95,6 @@ LayergroupController.prototype.layer = function(req, res, next) {
LayergroupController.prototype.tileOrLayer = function (req, res) {
var self = this;
console.log(req.context.user);
step(
function mapController$prepareParams() {
self.app.req2params(req, this);
@ -149,7 +155,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
global.statsClient.increment('windshaft.tiles.error');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
} else {
this.app.sendWithHeaders(res, tile, 200, headers);
this.sendResponse(req, res, [tile, headers, 200]);
global.statsClient.increment('windshaft.tiles.success');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
}
@ -206,8 +212,95 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
self.app.sendError(res, {errors: ['' + err] }, self.app.findStatusCode(err), 'STATIC_MAP', err);
} else {
res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format);
self.app.sendResponse(res, [image, 200]);
self.sendResponse(req, res, [image, 200]);
}
}
);
};
LayergroupController.prototype.sendResponse = function(req, res, args) {
var self = this;
res.header('Cache-Control', 'public,max-age=31536000');
// Set Last-Modified header
var lastUpdated;
if (req.params.cache_buster) {
// Assuming cache_buster is a timestamp
lastUpdated = new Date(parseInt(req.params.cache_buster));
} else {
lastUpdated = new Date();
}
res.header('Last-Modified', lastUpdated.toUTCString());
var dbName = req.params.dbname;
step(
function getAffectedTables() {
self.getAffectedTables(req.context.user, dbName, req.params.token, this);
},
function sendResponse(err, affectedTables) {
req.profiler.done('affectedTables');
if (err) {
console.log('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
}
self.app.sendResponse(res, args);
}
);
};
LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) {
return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId));
}
var self = this;
step(
function extractSQL() {
step(
function loadFromStore() {
self.mapStore.load(layergroupId, this);
},
function getSQL(err, mapConfig) {
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
return tableNames;
},
function finish(err, affectedTables) {
callback(err, affectedTables);
}
);
};

60
lib/cartodb/controllers/map.js
View File

@ -9,6 +9,7 @@ var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
@ -22,11 +23,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye
* @param metadataBackend
* @param {QueryTablesApi} queryTablesApi
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {{UserLimitsApi}} userLimitsApi
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor
*/
function MapController(app, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi,
surrogateKeysCache, userLimitsApi) {
surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
this.app = app;
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
@ -35,6 +37,8 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack
this.queryTablesApi = queryTablesApi;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTables = layergroupAffectedTables;
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
}
@ -147,13 +151,14 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
},
function afterLayergroupCreate(err, layergroup) {
assert.ifError(err);
self.afterLayergroupCreate(req, mapConfig, layergroup, this);
self.afterLayergroupCreate(req, res, mapConfig, layergroup, this);
},
function finish(err, layergroup) {
if (err) {
var statusCode = self.app.findStatusCode(err);
self.app.sendError(res, { errors: [ err.message ] }, statusCode, 'ANONYMOUS LAYERGROUP', err);
} else {
res.header('X-Layergroup-Id', layergroup.layergroupid);
self.app.sendResponse(res, [layergroup, 200]);
}
}
@ -169,6 +174,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
var mapConfig;
step(
function setupParams(){
self.app.req2params(req, this);
},
function getTemplateParams() {
prepareParamsFn(this);
},
@ -178,6 +186,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
self.queryTablesApi,
cdbuser,
req.params.template_id,
templateParams,
@ -197,7 +206,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
},
function afterLayergroupCreate(err, layergroup) {
assert.ifError(err);
self.afterLayergroupCreate(req, mapConfig, layergroup, this);
self.afterLayergroupCreate(req, res, mapConfig, layergroup, this);
},
function finishTemplateInstantiation(err, layergroup) {
if (err) {
@ -217,7 +226,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
};
MapController.prototype.afterLayergroupCreate = function(req, mapconfig, layergroup, callback) {
MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, callback) {
var self = this;
var username = req.context.user;
@ -258,34 +267,47 @@ MapController.prototype.afterLayergroupCreate = function(req, mapconfig, layergr
}).join(';');
var dbName = req.params.dbname;
var cacheKey = dbName + ':' + layergroup.layergroupid;
var layergroupId = layergroup.layergroupid;
step(
function getAffectedTablesAndLastUpdatedTime() {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
function checkCachedAffectedTables() {
return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
},
function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
assert.ifError(err);
if (hasCache) {
var next = this;
var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
if (err) {
return next(err);
}
return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
});
} else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
}
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
var cacheChannel = self.app.buildCacheChannel(dbName, result.affectedTables);
self.app.channelCache[cacheKey] = cacheChannel;
self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
var res = req.res;
if (res) {
if (req.method === 'GET') {
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.header('Last-Modified', (new Date()).toUTCString());
res.header('X-Cache-Channel', cacheChannel);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.header('Last-Modified', (new Date()).toUTCString());
res.header('X-Cache-Channel', tableCacheEntry.getCacheChannel());
if (result.affectedTables && result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tableCacheEntry);
}
res.header('X-Layergroup-Id', layergroup.layergroupid);
}
return null;

70
lib/cartodb/controllers/named_maps.js
View File

@ -4,18 +4,16 @@ var _ = require('underscore');
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var cors = require('../middleware/cors');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
function NamedMapsController(app, pgConnection, templateMaps, tileBackend, previewBackend, surrogateKeysCache,
tablesExtentApi, userLimitsApi) {
function NamedMapsController(app, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache,
tablesExtentApi) {
this.app = app;
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.namedMapProviderCache = namedMapProviderCache;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.tablesExtentApi = tablesExtentApi;
this.userLimitsApi = userLimitsApi;
}
module.exports = NamedMapsController;
@ -27,6 +25,46 @@ NamedMapsController.prototype.register = function(app) {
);
};
NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) {
this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName()));
res.header('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
res.header('Cache-Control', 'public,max-age=7200,must-revalidate');
var self = this;
var dbName = req.params.dbname;
step(
function getAffectedTablesAndLastUpdatedTime() {
namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
},
function sendResponse(err, result) {
req.profiler.done('affectedTables');
if (err) {
console.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.affectedTables) {
// we increase cache control as we can invalidate it
res.header('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.lastUpdatedTime);
} else {
lastModifiedDate = new Date();
}
res.header('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);
}
}
self.app.sendResponse(res, [resource, 200]);
}
);
};
NamedMapsController.prototype.tile = function(req, res) {
var self = this;
@ -38,10 +76,7 @@ NamedMapsController.prototype.tile = function(req, res) {
self.app.req2params(req, this);
},
function getTile() {
namedMapProvider = new NamedMapMapConfigProvider(
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
namedMapProvider = self.namedMapProviderCache.get(
cdbUser,
req.params.template_id,
req.query.config,
@ -60,10 +95,7 @@ NamedMapsController.prototype.tile = function(req, res) {
}
self.app.sendError(res, err, self.app.findStatusCode(err), 'NAMED_MAP_TILE', err);
} else {
self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName()));
res.setHeader('Content-Type', headers['Content-Type']);
res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate');
self.app.sendWithHeaders(res, tile, 200, headers);
self.sendResponse(req, res, tile, headers, namedMapProvider);
}
}
);
@ -85,10 +117,7 @@ NamedMapsController.prototype.staticMap = function(req, res) {
},
function getTemplate(err) {
assert.ifError(err);
namedMapProvider = new NamedMapMapConfigProvider(
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
namedMapProvider = self.namedMapProviderCache.get(
cdbUser,
req.params.template_id,
req.query.config,
@ -158,10 +187,7 @@ NamedMapsController.prototype.staticMap = function(req, res) {
}
self.app.sendError(res, err, self.app.findStatusCode(err), 'STATIC_VIZ_MAP', err);
} else {
self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName()));
res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format);
res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate');
self.app.sendResponse(res, [image, 200]);
self.sendResponse(req, res, image, headers, namedMapProvider);
}
}
);

23
lib/cartodb/controllers/named_maps_admin.js
View File

@ -5,9 +5,16 @@ var templateName = require('../backends/template_maps').templateName;
var cors = require('../middleware/cors');
function NamedMapsAdminController(app, templateMaps) {
/**
* @param app
* @param {TemplateMaps} templateMaps
* @param {AuthApi} authApi
* @constructor
*/
function NamedMapsAdminController(app, templateMaps, authApi) {
this.app = app;
this.templateMaps = templateMaps;
this.authApi = authApi;
}
module.exports = NamedMapsAdminController;
@ -28,7 +35,7 @@ NamedMapsAdminController.prototype.create = function(req, res) {
step(
function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this);
self.authApi.authorizedByAPIKey(cdbuser, req, this);
},
function addTemplate(err, authenticated) {
assert.ifError(err);
@ -53,7 +60,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
var tpl_id;
step(
function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this);
self.authApi.authorizedByAPIKey(cdbuser, req, this);
},
function updateTemplate(err, authenticated) {
assert.ifError(err);
@ -84,7 +91,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
var tpl_id;
step(
function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this);
self.authApi.authorizedByAPIKey(cdbuser, req, this);
},
function getTemplate(err, authenticated) {
assert.ifError(err);
@ -94,7 +101,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
self.templateMaps.getTemplate(cdbuser, tpl_id, this);
},
function prepareResponse(err, tpl_val) {
if ( err ) throw err;
assert.ifError(err);
if ( ! tpl_val ) {
err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
err.http_status = 404;
@ -120,7 +127,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
var tpl_id;
step(
function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this);
self.authApi.authorizedByAPIKey(cdbuser, req, this);
},
function deleteTemplate(err, authenticated) {
assert.ifError(err);
@ -130,7 +137,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
self.templateMaps.delTemplate(cdbuser, tpl_id, this);
},
function prepareResponse(err/*, tpl_val*/){
if ( err ) throw err;
assert.ifError(err);
return { status: 'ok' };
},
finishFn(self.app, res, 'DELETE TEMPLATE', ['', 204])
@ -147,7 +154,7 @@ NamedMapsAdminController.prototype.list = function(req, res) {
step(
function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this);
self.authApi.authorizedByAPIKey(cdbuser, req, this);
},
function listTemplates(err, authenticated) {
assert.ifError(err);

7
lib/cartodb/controllers/server_info.js
View File

@ -15,7 +15,7 @@ var versions = {
function ServerInfoController() {
this.healthConfig = global.environment.health || {};
this.healthCheck = new HealthCheck();
this.healthCheck = new HealthCheck(global.environment.disabled_file);
}
module.exports = ServerInfoController;
@ -37,13 +37,12 @@ ServerInfoController.prototype.version = function(req, res) {
ServerInfoController.prototype.health = function(req, res) {
if (!!this.healthConfig.enabled) {
var startTime = Date.now();
this.healthCheck.check(this.healthConfig, function(err, result) {
this.healthCheck.check(function(err) {
var ok = !err;
var response = {
enabled: true,
ok: ok,
elapsed: Date.now() - startTime,
result: result
elapsed: Date.now() - startTime
};
if (err) {
response.err = err.message;

45
lib/cartodb/models/mapconfig/named_map_provider.js
View File

@ -10,20 +10,25 @@ var templateName = require('../../backends/template_maps').templateName;
* @constructor
* @type {NamedMapMapConfigProvider}
*/
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, owner, templateId, config, authToken,
params) {
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi,
owner, templateId, config, authToken, params) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.owner = owner;
this.templateName = templateName(templateId);
this.config = config;
this.authToken = authToken;
this.params = params;
this.cacheBuster = Date.now();
// use template after call to mapConfig
this.template = null;
this.affectedTablesAndLastUpdate = null;
// providing
this.err = null;
this.mapConfig = null;
@ -144,7 +149,7 @@ NamedMapMapConfigProvider.prototype.getKey = function() {
};
NamedMapMapConfigProvider.prototype.getCacheBuster = function() {
return 0;
return this.cacheBuster;
};
NamedMapMapConfigProvider.prototype.filter = function(key) {
@ -153,7 +158,7 @@ NamedMapMapConfigProvider.prototype.filter = function(key) {
};
// Configure bases for cache keys suitable for string interpolation
var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}';
var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}';
var rendererKey = baseKey + ':{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}';
var baseKeyTpl = dot.template(baseKey);
@ -179,6 +184,8 @@ function configHash(config) {
return crypto.createHash('md5').update(JSON.stringify(config)).digest('hex').substring(0,8);
}
module.exports.configHash = configHash;
NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, callback) {
var self = this;
step(
@ -186,7 +193,7 @@ NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, call
self.pgConnection.setDBAuth(cdbuser, params, this);
},
function setConn(err) {
if ( err ) throw err;
assert.ifError(err);
self.pgConnection.setDBConn(cdbuser, params, this);
},
function finish(err) {
@ -198,3 +205,31 @@ NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, call
NamedMapMapConfigProvider.prototype.getTemplateName = function() {
return this.templateName;
};
NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) {
var self = this;
if (this.affectedTablesAndLastUpdate !== null) {
return callback(null, this.affectedTablesAndLastUpdate);
}
step(
function getMapConfig() {
self.getMapConfig(this);
},
function getSql(err, mapConfig) {
assert.ifError(err);
return mapConfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
},
function getAffectedTables(err, sql) {
assert.ifError(err);
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;
return callback(err, result);
}
);
};

21
lib/cartodb/monitoring/health_check.js
View File

@ -1,29 +1,20 @@
var fs = require('fs');
var step = require('step');
function HealthCheck() {
function HealthCheck(disableFile) {
this.disableFile = disableFile;
}
module.exports = HealthCheck;
HealthCheck.prototype.check = function(config, callback) {
HealthCheck.prototype.check = function(callback) {
var result = {
redis: {
ok: false
},
mapnik: {
ok: false
},
tile: {
ok: false
}
};
var self = this;
step(
function getManualDisable() {
fs.readFile(global.environment.disabled_file, this);
fs.readFile(self.disableFile, this);
},
function handleDisabledFile(err, data) {
var next = this;
@ -37,7 +28,7 @@ HealthCheck.prototype.check = function(config, callback) {
}
},
function handleResult(err) {
callback(err, result);
return callback(err);
}
);
};

411
lib/cartodb/server.js
View File

@ -19,6 +19,9 @@ var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
var NamedMapProviderCache = require('./cache/named_map_provider_cache');
var PgQueryRunner = require('./backends/pg_query_runner');
var PgConnection = require('./backends/pg_connection');
@ -73,12 +76,6 @@ module.exports = function(serverOptions) {
max_user_templates: global.environment.maxUserTemplates
});
// This is for Templated maps
//
// "named" is the official, "template" is for backward compatibility up to 1.6.x
//
var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
var surrogateKeysCacheBackends = [];
if (serverOptions.varnish_purge_enabled) {
@ -131,7 +128,6 @@ module.exports = function(serverOptions) {
pool: redisPool,
expire_time: serverOptions.grainstore.default_layergroup_ttl
});
app.mapStore = mapStore;
var onTileErrorStrategy;
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
@ -167,6 +163,16 @@ module.exports = function(serverOptions) {
var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
});
var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);
app.findStatusCode = function(err) {
var statusCode;
if ( err.http_status ) {
@ -195,7 +201,10 @@ module.exports = function(serverOptions) {
tileBackend,
previewBackend,
attributesBackend,
userLimitsApi
surrogateKeysCache,
userLimitsApi,
queryTablesApi,
layergroupAffectedTablesCache
).register(app);
new controller.Map(
@ -206,21 +215,20 @@ module.exports = function(serverOptions) {
metadataBackend,
queryTablesApi,
surrogateKeysCache,
userLimitsApi
userLimitsApi,
layergroupAffectedTablesCache
).register(app);
new controller.NamedMaps(
app,
pgConnection,
templateMaps,
namedMapProviderCache,
tileBackend,
previewBackend,
surrogateKeysCache,
tablesExtentApi,
userLimitsApi
tablesExtentApi
).register(app);
new controller.NamedMapsAdmin(app, templateMaps).register(app);
new controller.NamedMapsAdmin(app, templateMaps, authApi).register(app);
new controller.ServerInfo().register(app);
@ -241,100 +249,33 @@ module.exports = function(serverOptions) {
}
});
// GET routes for which we don't want to request any caching.
// POST/PUT/DELETE requests are never cached anyway.
var noCacheGETRoutes = [
'/',
'/version',
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
serverOptions.base_url_mapconfig,
serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
template_baseurl,
template_baseurl + '/:template_id',
template_baseurl + '/:template_id/jsonp'
];
app.sendResponse = function(res, args) {
var that = this;
var req = res.req;
var statusCode;
if ( res._windshaftStatusCode ) {
// Added by our override of sendError
statusCode = res._windshaftStatusCode;
} else {
if ( args.length > 2 ) statusCode = args[2];
else {
statusCode = args[1] || 200;
if (global.environment && global.environment.api_hostname) {
res.header('X-Served-By-Host', global.environment.api_hostname);
}
if (req && req.params && req.params.dbhost) {
res.header('X-Served-By-DB-Host', req.params.dbhost);
}
if ( req && req.profiler ) {
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
// res.send(body|status[, headers|status[, status]])
res.send.apply(res, args);
if ( req && req.profiler ) {
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
console.error("error sending profiling stats: " + err);
}
}
var req = res.req;
step (
function addCacheChannel() {
if ( ! req ) {
// having no associated request can happen when
// using fake response objects for testing layergroup
// creation
return false;
}
if ( ! req.params ) {
// service requests (/version, /)
// have no need for an X-Cache-Channel
return false;
}
if ( statusCode != 200 ) {
// We do not want to cache
// unsuccessful responses
return false;
}
if ( _.contains(noCacheGETRoutes, req.route.path) ) {
//console.log("Skipping cache channel in route:\n" + req.route.path);
return false;
}
//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" +
// mapCreateRoutes.join("\n"));
app.addCacheChannel(that, req, this);
},
function sendResponse(err/*, added*/) {
if ( err ) console.log(err + err.stack);
// When using custom results from tryFetch* methods,
// there is no "req" link in the result object.
// In those cases we don't want to send stats now
// as they will be sent at the real end of request
var req = res.req;
if (global.environment && global.environment.api_hostname) {
res.header('X-Served-By-Host', global.environment.api_hostname);
}
if (req && req.params && req.params.dbhost) {
res.header('X-Served-By-DB-Host', req.params.dbhost);
}
if ( req && req.profiler ) {
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
res.send.apply(res, args);
if ( req && req.profiler ) {
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
console.error("error sending profiling stats: " + err);
}
}
return null;
},
function finish(err) {
if ( err ) console.log(err + err.stack);
}
);
};
app.sendWithHeaders = function(res, what, status, headers) {
app.sendResponse(res, [what, headers, status]);
};
app.sendError = function(res, err, statusCode, label, tolog) {
@ -453,9 +394,9 @@ module.exports = function(serverOptions) {
step(
function getPrivacy(){
app.authorize(req, this);
authApi.authorize(req, this);
},
function gatekeep(err, authorized){
function validateAuthorization(err, authorized) {
if (req.profiler) {
req.profiler.done('authorize');
}
@ -472,7 +413,9 @@ module.exports = function(serverOptions) {
pgConnection.setDBConn(user, req.params, this);
},
function finishSetup(err) {
if ( err ) { callback(err, req); return; }
if ( err ) {
return callback(err, req);
}
// Add default database connection parameters
// if none given
@ -488,260 +431,6 @@ module.exports = function(serverOptions) {
);
};
// TODO: review lifetime of elements of this cache
// NOTE: by-token indices should only be dropped when
// the corresponding layegroup is dropped, because
// we have no SQL after layer creation.
app.channelCache = {};
app.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};
app.generateCacheChannel = function(app, req, callback){
// Build channelCache key
var dbName = req.params.dbname;
var cacheKey = [ dbName, req.params.token ].join(':');
// no token means no tables associated
if (!req.params.token) {
return callback(null, this.buildCacheChannel(dbName, []));
}
step(
function checkCached() {
if ( app.channelCache.hasOwnProperty(cacheKey) ) {
return callback(null, app.channelCache[cacheKey]);
}
return null;
},
function extractSQL(err) {
assert.ifError(err);
// TODO: cached cache channel for token-based access should
// be constructed at renderer cache creation time
// See http://github.com/CartoDB/Windshaft-cartodb/issues/152
if ( ! app.mapStore ) {
throw new Error('missing channel cache for token ' + req.params.token);
}
var mapStore = app.mapStore;
step(
function loadFromStore() {
mapStore.load(req.params.token, this);
},
function getSQL(err, mapConfig) {
if (req.profiler) {
req.profiler.done('mapStore_load');
}
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
assert.ifError(err);
if (req.profiler) {
req.profiler.done('affectedTables');
}
var cacheChannel = app.buildCacheChannel(dbName,tableNames);
app.channelCache[cacheKey] = cacheChannel;
return cacheChannel;
},
function finish(err, cacheChannel) {
callback(err, cacheChannel);
}
);
};
// Set the cache chanel info to invalidate the cache on the frontend server
//
// @param req The request object.
// The function will have no effect unless req.res exists.
// It is expected that req.params contains 'table' and 'dbname'
//
// @param cb function(err, channel) will be called when ready.
// the channel parameter will be null if nothing was added
//
app.addCacheChannel = function(app, req, cb) {
// skip non-GET requests, or requests for which there's no response
if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
if (req.profiler) {
req.profiler.start('addCacheChannel');
}
var res = req.res;
if ( req.params.token ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
var ttl = global.environment.varnish.ttl || 86400;
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
}
// Set Last-Modified header
var lastUpdated;
if ( req.params.cache_buster ) {
// Assuming cache_buster is a timestamp
// FIXME: store lastModified in the cache channel instead
lastUpdated = new Date(parseInt(req.params.cache_buster));
} else {
lastUpdated = new Date();
}
res.header('Last-Modified', lastUpdated.toUTCString());
app.generateCacheChannel(app, req, function(err, channel){
if (req.profiler) {
req.profiler.done('generateCacheChannel');
req.profiler.end();
}
if ( ! err ) {
res.header('X-Cache-Channel', channel);
cb(null, channel);
} else {
console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
// TODO: evaluate if we should bubble up the error instead
cb(null, 'ERROR');
}
});
};
// Check if a request is authorized by a signer
//
// @param req express request object
// @param callback function(err, signed_by) signed_by will be
// null if the request is not signed by anyone
// or will be a string cartodb username otherwise.
//
app.authorizedBySigner = function(req, callback) {
if ( ! req.params.token || ! req.params.signer ) {
return callback(null, false); // no signer requested
}
var layergroup_id = req.params.token;
var auth_token = req.params.auth_token;
mapStore.load(layergroup_id, function(err, mapConfig) {
if (err) {
return callback(err);
}
var authorized = templateMaps.isAuthorized(mapConfig.obj().template, auth_token);
return callback(null, authorized);
});
};
// Check if a request is authorized by api_key
//
// @param user
// @param req express request object
// @param callback function(err, authorized)
// NOTE: authorized is expected to be 0 or 1 (integer)
//
app.authorizedByAPIKey = function(user, req, callback) {
var givenKey = req.query.api_key || req.query.map_key;
if ( ! givenKey && req.body ) {
// check also in request body
givenKey = req.body.api_key || req.body.map_key;
}
if ( ! givenKey ) {
return callback(null, 0); // no api key, no authorization...
}
step(
function () {
metadataBackend.getUserMapKey(user, this);
},
function checkApiKey(err, val){
assert.ifError(err);
return val && givenKey == val;
},
function finish(err, authorized) {
callback(err, authorized);
}
);
};
/**
* Check access authorization
*
* @param req - standard req object. Importantly contains table and host information
* @param callback function(err, allowed) is access allowed not?
*/
app.authorize = function(req, callback) {
var self = this;
var user = req.context.user;
step(
function () {
self.authorizedByAPIKey(user, req, this);
},
function checkApiKey(err, authorized){
if (req.profiler) {
req.profiler.done('authorizedByAPIKey');
}
assert.ifError(err);
// if not authorized by api_key, continue
if (!authorized) {
// not authorized by api_key, check if authorized by signer
return self.authorizedBySigner(req, this);
}
// authorized by api key, login as the given username and stop
pgConnection.setDBAuth(user, req.params, function(err) {
callback(err, true); // authorized (or error)
});
},
function checkSignAuthorized(err, authorized) {
if (err) {
return callback(err);
}
if ( ! authorized ) {
// request not authorized by signer.
// if no signer name was given, let dbparams and
// PostgreSQL do the rest.
//
if ( ! req.params.signer ) {
return callback(null, true); // authorized so far
}
// if signer name was given, return no authorization
return callback(null, false);
}
pgConnection.setDBAuth(user, req.params, function(err) {
if (req.profiler) {
req.profiler.done('setDBAuth');
}
callback(err, true); // authorized (or error)
});
}
);
};
return app;
};

1
lib/cartodb/server_options.js
View File

@ -62,6 +62,7 @@ module.exports = {
},
renderer: {
mapnik: rendererConfig.mapnik,
torque: rendererConfig.torque,
http: rendererConfig.http
},
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161

1048
npm-shrinkwrap.json generated
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "2.7.2",
"version": "2.12.1",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@ -33,6 +33,7 @@
"cartodb-psql": "~0.4.0",
"fastly-purge": "~1.0.0",
"redis-mpool": "~0.4.0",
"lru-cache": "2.6.5",
"lzma": "~1.3.7",
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
},

2
test/acceptance/health_check.js
View File

@ -60,7 +60,7 @@ describe('health checks', function () {
callback(null, "Maintenance");
};
healthCheck.check(null, function(err/*, result*/) {
healthCheck.check(function(err) {
assert.equal(err.message, "Maintenance");
assert.equal(err.http_status, 503);
done();

302
test/acceptance/multilayer.js
View File

@ -17,6 +17,8 @@ var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
var suiteName = 'multilayer:postgres=layergroup_url=' + layergroup_url;
@ -67,17 +69,14 @@ suite(suiteName, function() {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.last_updated, expected_last_updated);
if ( expected_token ) {
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
}
else expected_token = parsedBody.layergroupid.split(':')[0];
assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
expected_token = parsedBody.layergroupid.split(':')[0];
next(null, res);
});
},
function do_get_tile(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
@ -122,7 +121,7 @@ suite(suiteName, function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/170
function do_get_tile_nosignature(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + '/localhost@' + expected_token + ':cb0/0/0/0.png',
@ -139,7 +138,7 @@ suite(suiteName, function() {
},
function do_get_grid_layer0(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json',
@ -156,7 +155,7 @@ suite(suiteName, function() {
},
function do_get_grid_layer1(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json',
@ -178,12 +177,20 @@ suite(suiteName, function() {
console.log("Error: " + err);
}
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
if ( err ) {
errors.push(err.message);
}
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
});
});
}
@ -226,17 +233,23 @@ suite(suiteName, function() {
test("get creation requests has cache", function(done) {
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
' from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.0.1'
} }
]
};
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select cartodb_id, the_geom_webmercator from test_table',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.0.1',
interactivity: 'cartodb_id'
} },
{ options: {
sql: 'select cartodb_id, the_geom_webmercator from test_table_2',
cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
cartocss_version: '2.0.2',
interactivity: 'cartodb_id'
} }
]
};
var expected_token;
step(
@ -250,11 +263,15 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); });
},
function do_check_create(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
'public.test_table',
'public.test_table_2'
]).key().join(' '));
return null;
},
function finish(err) {
@ -264,12 +281,20 @@ suite(suiteName, function() {
console.log("Error: " + err);
}
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
if ( err ) {
errors.push(err.message);
}
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
});
});
}
@ -353,13 +378,15 @@ suite(suiteName, function() {
if ( expected_token ) {
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
}
else expected_token = parsedBody.layergroupid.split(':')[0];
else {
expected_token = parsedBody.layergroupid.split(':')[0];
}
next(null, res);
});
},
function do_get_tile1(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb10/1/0/0.png',
@ -398,7 +425,7 @@ suite(suiteName, function() {
},
function do_get_tile4(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb11/4/0/0.png',
@ -437,7 +464,7 @@ suite(suiteName, function() {
},
function do_get_grid1(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/1/0/0.grid.json',
@ -454,7 +481,7 @@ suite(suiteName, function() {
},
function do_get_grid4(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/4/0/0.grid.json',
@ -476,12 +503,20 @@ suite(suiteName, function() {
console.log("Error: " + err);
}
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
if ( err ) {
errors.push(err.message);
}
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
});
});
}
@ -511,13 +546,17 @@ suite(suiteName, function() {
{
var next = this;
redis_stats_client.select(redis_stats_db, function(err) {
if ( err ) next(err);
else redis_stats_client.del(statskey+':global', next);
if ( err ) {
next(err);
}
else {
redis_stats_client.del(statskey+':global', next);
}
});
},
function do_post_1(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url,
@ -531,12 +570,12 @@ suite(suiteName, function() {
});
},
function check_global_stats_1(err, val) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(val, 1, "Expected score of " + now + " in " + statskey + ":global to be 1, got " + val);
redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this);
},
function check_tag_stats_1_do_post_2(err, val) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(val, 1, "Expected score of " + now + " in " + statskey + ":stat_tag:" + layergroup.stat_tag +
" to be 1, got " + val);
var next = this;
@ -553,19 +592,21 @@ suite(suiteName, function() {
},
function check_global_stats_2(err, val)
{
if ( err ) throw err;
assert.ifError(err);
assert.equal(val, 2, "Expected score of " + now + " in " + statskey + ":global to be 2, got " + val);
redis_stats_client.zscore(statskey+':stat_tag:' + layergroup.stat_tag, now, this);
},
function check_tag_stats_2(err, val)
{
if ( err ) throw err;
assert.ifError(err);
assert.equal(val, 2, "Expected score of " + now + " in " + statskey + ":stat_tag:" + layergroup.stat_tag +
" to be 2, got " + val);
return 1;
},
function cleanup_map_style(err) {
if ( err ) errors.push('' + err);
if ( err ) {
errors.push('' + err);
}
var next = this;
// trip epoch
expected_token = expected_token.split(':')[0];
@ -574,13 +615,21 @@ suite(suiteName, function() {
});
},
function cleanup_stats(err) {
if ( err ) errors.push('' + err);
if ( err ) {
errors.push('' + err);
}
redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this);
},
function finish(err) {
if ( err ) errors.push('' + err);
if ( errors.length ) done(new Error(errors.join(',')));
else done(null);
if ( err ) {
errors.push('' + err);
}
if ( errors.length ) {
done(new Error(errors.join(',')));
}
else {
done(null);
}
}
);
});
@ -678,13 +727,15 @@ suite(suiteName, function() {
if ( expected_token ) {
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
}
else expected_token = parsedBody.layergroupid.split(':')[0];
else {
expected_token = parsedBody.layergroupid.split(':')[0];
}
next(null, res);
});
},
function do_get_tile(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234',
@ -705,7 +756,7 @@ suite(suiteName, function() {
},
function do_get_grid_layer0(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json?map_key=1234',
@ -718,7 +769,7 @@ suite(suiteName, function() {
},
function do_get_grid_layer1(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json?map_key=1234',
@ -732,7 +783,7 @@ suite(suiteName, function() {
},
function do_get_tile_unauth(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
@ -748,7 +799,7 @@ suite(suiteName, function() {
},
function do_get_grid_layer0_unauth(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json',
@ -763,7 +814,7 @@ suite(suiteName, function() {
},
function do_get_grid_layer1_unauth(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json',
@ -783,12 +834,20 @@ suite(suiteName, function() {
console.log("Error: " + err);
}
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
if ( err ) {
errors.push(err.message);
}
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
});
});
}
@ -823,19 +882,21 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); });
},
function check_post(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.last_updated, expected_last_updated);
if ( expected_token ) {
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
}
else expected_token = parsedBody.layergroupid.split(':')[0];
else {
expected_token = parsedBody.layergroupid.split(':')[0];
}
return null;
},
function do_get0(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234',
@ -845,7 +906,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); });
},
function do_check0(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -857,14 +918,14 @@ suite(suiteName, function() {
return null;
},
function do_restart_server(err/*, res*/) {
if ( err ) throw err;
assert.ifError(err);
// hack simulating restart...
server = new CartodbWindshaft(serverOptions);
return null;
},
function do_get1(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234',
@ -874,7 +935,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); });
},
function do_check1(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -892,12 +953,20 @@ suite(suiteName, function() {
console.log("Error: " + err);
}
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors.join(',')));
else done(null);
if ( err ) {
errors.push(err.message);
}
if ( errors.length ) {
done(new Error(errors.join(',')));
}
else {
done(null);
}
});
});
}
@ -1024,7 +1093,7 @@ suite(suiteName, function() {
},
function do_get_tile(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
@ -1048,12 +1117,20 @@ suite(suiteName, function() {
console.log("Error: " + err);
}
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
if ( err ) {
errors.push(err.message);
}
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
});
});
}
@ -1087,7 +1164,7 @@ suite(suiteName, function() {
}, {}, function(res) { next(null, res); });
},
function check_result(err, res) {
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsedBody = JSON.parse(res.body);
@ -1104,7 +1181,7 @@ suite(suiteName, function() {
},
function do_get_tile(err)
{
if ( err ) throw err;
assert.ifError(err);
var next = this;
assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?api_key=1234',
@ -1114,19 +1191,27 @@ suite(suiteName, function() {
}, {}, function(res) { next(null, res); });
},
function check_get_tile(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 200, res.body);
return null;
},
function cleanup(err) {
if ( err ) errors.push(err.message);
if ( ! expected_token ) return null;
if ( err ) {
errors.push(err.message);
}
if ( ! expected_token ) {
return null;
}
var next = this;
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
next();
});
});
@ -1136,8 +1221,12 @@ suite(suiteName, function() {
errors.push(err.message);
console.log("Error: " + err);
}
if ( errors.length ) done(new Error(errors));
else done(null);
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
}
);
});
@ -1146,11 +1235,14 @@ suite(suiteName, function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/111
test("sql string can be very long", function(done){
var long_val = 'pretty';
for (var i=0; i<1024; ++i) long_val += ' long';
for (var i=0; i<1024; ++i) {
long_val += ' long';
}
long_val += ' string';
var sql = "SELECT ";
for (i=0; i<16; ++i)
for (i=0; i<16; ++i) {
sql += "'" + long_val + "'::text as pretty_long_field_name_" + i + ", ";
}
sql += "cartodb_id, the_geom_webmercator FROM gadm4 g";
var layergroup = {
version: '1.0.0',
@ -1178,7 +1270,7 @@ suite(suiteName, function() {
}, {}, function(res) { next(null, res); });
},
function check_result(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsedBody = JSON.parse(res.body);
var token_components = parsedBody.layergroupid.split(':');
@ -1186,22 +1278,36 @@ suite(suiteName, function() {
return null;
},
function cleanup(err) {
if ( err ) errors.push('' + err);
if ( ! expected_token ) return null;
if ( err ) {
errors.push('' + err);
}
if ( ! expected_token ) {
return null;
}
var next = this;
redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( err ) {
errors.push(err.message);
}
next();
});
});
},
function finish(err) {
if ( err ) errors.push('' + err);
if ( errors.length ) done(new Error(errors.join(',')));
else done(null);
if ( err ) {
errors.push('' + err);
}
if ( errors.length ) {
done(new Error(errors.join(',')));
}
else {
done(null);
}
}
);
});
@ -1232,7 +1338,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); });
},
function check_post(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed));
@ -1274,7 +1380,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); });
},
function check_post(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed));

4
test/acceptance/multilayer_server.js
View File

@ -318,7 +318,7 @@ describe('tests from old api translated to multilayer', function() {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed, {
errors: ["Error: could not fetch affected tables and last updated time: fake error message"]
errors: ["Error: could not fetch affected tables or last updated time: fake error message"]
});
done();
@ -346,7 +346,7 @@ describe('tests from old api translated to multilayer', function() {
};
// reset internal cacheChannel cache
server.channelCache = {};
server.layergroupAffectedTablesCache.cache.reset();
assert.response(server,
{

4
test/acceptance/ported/support/ported_server_options.js
View File

@ -58,6 +58,10 @@ module.exports = _.extend({}, serverOptions, {
_.extend(req.params, req.query);
req.params.user = 'localhost';
req.context = {user: 'localhost'};
req.params.dbhost = global.environment.postgres.host;
req.params.dbport = req.params.dbport || global.environment.postgres.port;
req.params.dbuser = 'test_windshaft_publicuser';
if (req.params.dbname !== 'windshaft_test2') {
req.params.dbuser = 'test_windshaft_cartodb_user_1';

2
test/acceptance/server.js
View File

@ -23,7 +23,7 @@ suite('server', function() {
},{}, function(res, err) { next(err,res); });
},
function doCheck(err, res) {
if ( err ) throw err;
assert.ifError(err);
assert.ok(res.statusCode, 200);
var cc = res.headers['x-cache-channel'];
assert.ok(!cc);

453
test/acceptance/templates.js
View File

File diff suppressed because it is too large Load Diff

4
test/support/sql/CDB_QueryTables.sql
View File

@ -41,11 +41,11 @@ BEGIN
xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
)
SELECT unnest(x) as p, unnest(s) as sc from inp
SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
LOOP
-- RAISE DEBUG 'tab: %', rec2.p;
-- RAISE DEBUG 'sc: %', rec2.sc;
tables := array_append(tables, (rec2.sc || '.' || rec2.p));
tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;

1
test/support/test_helper.js
View File

@ -34,6 +34,7 @@ function lzma_compress_to_base64(payload, mode, callback) {
// Throws on failure
function checkNoCache(res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
assert.ok(!res.headers.hasOwnProperty('surrogate-key'));
assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ?
assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ?
}

77
test/unit/cartodb/template_maps.test.js
View File

@ -202,7 +202,7 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this);
},
function addOmonimousTemplate(err, id) {
if ( err ) throw err;
assert.ifError(err);
tpl_id = id;
assert.equal(tpl_id, 'first');
expected_failure = true;
@ -210,13 +210,15 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this);
},
function getTemplate(err) {
if ( ! expected_failure && err ) throw err;
if ( ! expected_failure && err ) {
throw err;
}
assert.ok(err);
assert.ok(err.message.match(/already exists/i), err);
tmap.getTemplate('me', tpl_id, this);
},
function delTemplate(err, got_tpl) {
if ( err ) throw err;
assert.ifError(err);
assert.deepEqual(got_tpl, _.extend({}, tpl, {auth: {method: 'open'}, placeholders: {}}));
tmap.delTemplate('me', tpl_id, this);
},
@ -238,31 +240,35 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl1, this);
},
function addTemplate2(err, id) {
if ( err ) throw err;
assert.ifError(err);
tpl1_id = id;
tmap.addTemplate('me', tpl2, this);
},
function listTemplates(err, id) {
if ( err ) throw err;
assert.ifError(err);
tpl2_id = id;
tmap.listTemplates('me', this);
},
function checkTemplates(err, ids) {
if ( err ) throw err;
assert.ifError(err);
assert.equal(ids.length, 2);
assert.ok(ids.indexOf(tpl1_id) != -1, ids.join(','));
assert.ok(ids.indexOf(tpl2_id) != -1, ids.join(','));
assert.ok(ids.indexOf(tpl1_id) !== -1, ids.join(','));
assert.ok(ids.indexOf(tpl2_id) !== -1, ids.join(','));
return null;
},
function delTemplate1(err) {
if ( tpl1_id ) {
var next = this;
tmap.delTemplate('me', tpl1_id, function(e) {
if ( err || e ) next(new Error(err + '; ' + e));
else next();
if ( err || e ) {
next(new Error(err + '; ' + e));
}
else {
next();
}
});
} else {
if ( err ) throw err;
assert.ifError(err);
return null;
}
},
@ -270,11 +276,15 @@ describe('template_maps', function() {
if ( tpl2_id ) {
var next = this;
tmap.delTemplate('me', tpl2_id, function(e) {
if ( err || e ) next(new Error(err + '; ' + e));
else next();
if ( err || e ) {
next(new Error(err + '; ' + e));
}
else {
next();
}
});
} else {
if ( err ) throw err;
assert.ifError(err);
return null;
}
},
@ -301,14 +311,16 @@ describe('template_maps', function() {
},
// Updating template name should fail
function updateTemplateName(err, id) {
if ( err ) throw err;
assert.ifError(err);
tpl_id = id;
expected_failure = true;
tpl.name = 'second';
tmap.updTemplate(owner, tpl_id, tpl, this);
},
function updateTemplateAuth(err) {
if ( err && ! expected_failure) throw err;
if ( err && ! expected_failure) {
throw err;
}
expected_failure = false;
assert.ok(err);
tpl.name = 'first';
@ -317,13 +329,15 @@ describe('template_maps', function() {
tmap.updTemplate(owner, tpl_id, tpl, this);
},
function updateTemplateWithInvalid(err) {
if ( err ) throw err;
assert.ifError(err);
tpl.version = '999.999.999';
expected_failure = true;
tmap.updTemplate(owner, tpl_id, tpl, this);
},
function updateUnexistentTemplate(err) {
if ( err && ! expected_failure) throw err;
if ( err && ! expected_failure) {
throw err;
}
assert.ok(err);
assert.ok(err.message.match(/unsupported.*version/i), err);
tpl.version = '0.0.1';
@ -331,7 +345,9 @@ describe('template_maps', function() {
tmap.updTemplate(owner, 'unexistent', tpl, this);
},
function delTemplate(err) {
if ( err && ! expected_failure) throw err;
if ( err && ! expected_failure) {
throw err;
}
expected_failure = false;
assert.ok(err);
assert.ok(err.message.match(/cannot update name/i), err);
@ -344,6 +360,7 @@ describe('template_maps', function() {
});
it('instanciate templates', function() {
// jshint maxcomplexity:7
var tmap = new TemplateMaps(redis_pool);
assert.ok(tmap);
@ -456,14 +473,14 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this);
},
function twoForMe(err, id) {
if ( err ) throw err;
assert.ifError(err);
assert.ok(id);
idMe.push(id);
tpl.name = 'twoForMe';
tmap.addTemplate('me', tpl, this);
},
function threeForMe(err, id) {
if ( err ) throw err;
assert.ifError(err);
assert.ok(id);
idMe.push(id);
tpl.name = 'threeForMe';
@ -471,37 +488,39 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this);
},
function errForMe(err/*, id*/) {
if ( err && ! expectErr ) throw err;
if ( err && ! expectErr ) {
throw err;
}
expectErr = false;
assert.ok(err);
assert.ok(err.message.match(/limit.*template/), err);
return null;
},
function delOneMe(err) {
if ( err ) throw err;
assert.ifError(err);
tmap.delTemplate('me', idMe.shift(), this);
},
function threeForMeRetry(err) {
if ( err ) throw err;
assert.ifError(err);
tpl.name = 'threeForMe';
tmap.addTemplate('me', tpl, this);
},
function oneForYou(err, id) {
if ( err ) throw err;
assert.ifError(err);
assert.ok(id);
idMe.push(id);
tpl.name = 'oneForYou';
tmap.addTemplate('you', tpl, this);
},
function twoForYou(err, id) {
if ( err ) throw err;
assert.ifError(err);
assert.ok(id);
idYou.push(id);
tpl.name = 'twoForYou';
tmap.addTemplate('you', tpl, this);
},
function threeForYou(err, id) {
if ( err ) throw err;
assert.ifError(err);
assert.ok(id);
idYou.push(id);
tpl.name = 'threeForYou';
@ -509,7 +528,9 @@ describe('template_maps', function() {
tmap.addTemplate('you', tpl, this);
},
function errForYou(err/*, id*/) {
if ( err && ! expectErr ) throw err;
if ( err && ! expectErr ) {
throw err;
}
expectErr = false;
assert.ok(err);
assert.ok(err.message.match(/limit.*template/), err);