cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'standalone-server' into standalone-server-mvt

Browse Source 
Conflicts:
	npm-shrinkwrap.json
This commit is contained in:
Raul Ochoa 2015-09-08 15:46:23 +02:00
commit b2e1e5361f
37 changed files with 2169 additions and 1117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.jshintrc
View File

@ -7,8 +7,8 @@
// // Enforcing // // Enforcing
// "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.) // "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
// "camelcase" : false, // true: Identifiers must be in camelCase // "camelcase" : false, // true: Identifiers must be in camelCase
// "curly" : true, // true: Require {} for every new block or scope "curly" : true, // true: Require {} for every new block or scope
// "eqeqeq" : true, // true: Require triple equals (===) for comparison "eqeqeq" : true, // true: Require triple equals (===) for comparison
"forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty() "forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty()
"freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc. "freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc.
"immed" : true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());` "immed" : true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
@ -31,7 +31,7 @@
// "maxparams" : false, // {int} Max number of formal params allowed per function // "maxparams" : false, // {int} Max number of formal params allowed per function
// "maxdepth" : false, // {int} Max depth of nested blocks (within functions) // "maxdepth" : false, // {int} Max depth of nested blocks (within functions)
// "maxstatements" : false, // {int} Max number statements per function // "maxstatements" : false, // {int} Max number statements per function
"maxcomplexity" : 8, // {int} Max cyclomatic complexity per function "maxcomplexity" : 6, // {int} Max cyclomatic complexity per function
"maxlen" : 120, // {int} Max number of characters per line "maxlen" : 120, // {int} Max number of characters per line
// //
// // Relaxing // // Relaxing

12
Makefile
View File

@ -25,13 +25,9 @@ test: config/environments/test.js
test/unit/cartodb/cache/model/*.js \ test/unit/cartodb/cache/model/*.js \
test/integration/*.js \ test/integration/*.js \
test/acceptance/*.js \ test/acceptance/*.js \
test/acceptance/cache/*.js test/acceptance/cache/*.js \
test/acceptance/ported/*.js \
test-ported: config/environments/test.js test/unit/cartodb/ported/*.js
@echo "***tests ported***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \
test/unit/cartodb/ported/*.js \
test/acceptance/ported/*.js
test-unit: config/environments/test.js test-unit: config/environments/test.js
@echo "***tests***" @echo "***tests***"
@ -54,7 +50,7 @@ jshint:
@echo "***jshint***" @echo "***jshint***"
@./node_modules/.bin/jshint lib/ test/ app.js @./node_modules/.bin/jshint lib/ test/ app.js
test-all: jshint test test-ported test-all: jshint test
coverage: coverage:
@RUNTESTFLAGS=--with-coverage make test @RUNTESTFLAGS=--with-coverage make test

58
NEWS.md
View File

@ -1,8 +1,64 @@
# Changelog # Changelog
## 2.12.1
Released 2015-mm-dd
## 2.12.0
Released 2015-08-27
Announcements:
- Upgrades windshaft to [0.51.0](https://github.com/CartoDB/Windshaft/releases/tag/0.51.0)
New features:
- Make http and https globalAgent options configurable
* If config is not provided it configures them with default values
## 2.11.0
Released 2015-08-26
Announcements:
- Upgrades windshaft to [0.50.0](https://github.com/CartoDB/Windshaft/releases/tag/0.50.0)
## 2.10.0
Released 2015-08-18
New features:
- Exposes metatile cache configuration for tilelive-mapnik, see configuration sample files for more information.
Announcements:
- Upgrades windshaft to [0.49.0](https://github.com/CartoDB/Windshaft/releases/tag/0.49.0)
## 2.9.0
Released 2015-08-06
New features:
- Send memory usage stats
## 2.8.0
Released 2015-07-15
Announcements:
- Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0)
## 2.7.2 ## 2.7.2
Released 2015-mm-dd Released 2015-07-14
Enhancements:
- Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names
## 2.7.1 ## 2.7.1

30
app.js
View File

@ -1,6 +1,10 @@
var http = require('http');
var https = require('https');
var path = require('path'); var path = require('path');
var fs = require('fs'); var fs = require('fs');
var _ = require('underscore');
var ENVIRONMENT; var ENVIRONMENT;
if ( process.argv[2] ) { if ( process.argv[2] ) {
ENVIRONMENT = process.argv[2]; ENVIRONMENT = process.argv[2];
@ -38,6 +42,17 @@ if (global.environment.uv_threadpool_size) {
process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size; process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
} }
// set global HTTP and HTTPS agent default configurations
// ref https://nodejs.org/api/http.html#http_new_agent_options
var agentOptions = _.defaults(global.environment.httpAgent || {}, {
keepAlive: false,
keepAliveMsecs: 1000,
maxSockets: Infinity,
maxFreeSockets: 256
});
http.globalAgent = new http.Agent(agentOptions);
https.globalAgent = new https.Agent(agentOptions);
if ( global.environment.log_filename ) { if ( global.environment.log_filename ) {
var logdir = path.dirname(global.environment.log_filename); var logdir = path.dirname(global.environment.log_filename);
// See cwd inlog4js.configure call below // See cwd inlog4js.configure call below
@ -79,12 +94,19 @@ server.listen(serverOptions.bind.port, serverOptions.bind.host);
var version = require("./package").version; var version = require("./package").version;
server.on('listening', function() { server.on('listening', function() {
console.log( console.log(
"Windshaft tileserver %s started on %s:%s (%s)", "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
version, serverOptions.bind.host, serverOptions.bind.port, ENVIRONMENT version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
); );
}); });
setInterval(function() {
var memoryUsage = process.memoryUsage();
Object.keys(memoryUsage).forEach(function(k) {
global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
});
}, 5000);
process.on('SIGHUP', function() { process.on('SIGHUP', function() {
global.log4js.clearAndShutdownAppenders(function() { global.log4js.clearAndShutdownAppenders(function() {
global.log4js.configure(log4js_config); global.log4js.configure(log4js_config);

37
config/environments/development.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'development' environment: 'development'
,port: 8181 ,port: 8181
,host: '127.0.0.1' ,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined ,uv_threadpool_size: undefined
// Regular expression pattern to extract username // Regular expression pattern to extract username
// from hostname. Must have a single grabbing block. // from hostname. Must have a single grabbing block.
@ -86,8 +89,10 @@ var config = {
cache_ttl: 60000, cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: { mapnik: {
// The size of the pool of internal mapnik renderers // The size of the pool of internal mapnik backend
// Check the configuration of uv_threadpool_size to use suitable value // This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8, poolSize: 8,
// Metatile is the number of tiles-per-side that are going // Metatile is the number of tiles-per-side that are going
@ -96,6 +101,17 @@ var config = {
// wasted time. // wasted time.
metatile: 2, metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format // Override metatile behaviour depending on the format
formatMetatile: { formatMetatile: {
png: 2, png: 2,
@ -146,6 +162,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png' src: __dirname + '/../../assets/default-placeholder.png'
} }
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
} }
} }
,millstone: { ,millstone: {
@ -180,6 +206,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
} }
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: { ,varnish: {
host: 'localhost', host: 'localhost',
port: 6082, // the por for the telnet interface where varnish is listening to port: 6082, // the por for the telnet interface where varnish is listening to

37
config/environments/production.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'production' environment: 'production'
,port: 8181 ,port: 8181
,host: '127.0.0.1' ,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined ,uv_threadpool_size: undefined
// Regular expression pattern to extract username // Regular expression pattern to extract username
// from hostname. Must have a single grabbing block. // from hostname. Must have a single grabbing block.
@ -80,8 +83,10 @@ var config = {
cache_ttl: 60000, cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: { mapnik: {
// The size of the pool of internal mapnik renderers // The size of the pool of internal mapnik backend
// Check the configuration of uv_threadpool_size to use suitable value // This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8, poolSize: 8,
// Metatile is the number of tiles-per-side that are going // Metatile is the number of tiles-per-side that are going
@ -90,6 +95,17 @@ var config = {
// wasted time. // wasted time.
metatile: 2, metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format // Override metatile behaviour depending on the format
formatMetatile: { formatMetatile: {
png: 2, png: 2,
@ -140,6 +156,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png' src: __dirname + '/../../assets/default-placeholder.png'
} }
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
} }
} }
,millstone: { ,millstone: {
@ -174,6 +200,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
} }
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: { ,varnish: {
host: 'localhost', host: 'localhost',
port: 6082, // the por for the telnet interface where varnish is listening to port: 6082, // the por for the telnet interface where varnish is listening to

37
config/environments/staging.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'production' environment: 'production'
,port: 8181 ,port: 8181
,host: '127.0.0.1' ,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined ,uv_threadpool_size: undefined
// Regular expression pattern to extract username // Regular expression pattern to extract username
// from hostname. Must have a single grabbing block. // from hostname. Must have a single grabbing block.
@ -80,8 +83,10 @@ var config = {
cache_ttl: 60000, cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: { mapnik: {
// The size of the pool of internal mapnik renderers // The size of the pool of internal mapnik backend
// Check the configuration of uv_threadpool_size to use suitable value // This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8, poolSize: 8,
// Metatile is the number of tiles-per-side that are going // Metatile is the number of tiles-per-side that are going
@ -90,6 +95,17 @@ var config = {
// wasted time. // wasted time.
metatile: 2, metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format // Override metatile behaviour depending on the format
formatMetatile: { formatMetatile: {
png: 2, png: 2,
@ -140,6 +156,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png' src: __dirname + '/../../assets/default-placeholder.png'
} }
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
} }
} }
,millstone: { ,millstone: {
@ -174,6 +200,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
} }
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: { ,varnish: {
host: 'localhost', host: 'localhost',
port: 6082, // the por for the telnet interface where varnish is listening to port: 6082, // the por for the telnet interface where varnish is listening to

37
config/environments/test.js.example
View File

@ -2,6 +2,9 @@ var config = {
environment: 'test' environment: 'test'
,port: 8888 ,port: 8888
,host: '127.0.0.1' ,host: '127.0.0.1'
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined ,uv_threadpool_size: undefined
// Regular expression pattern to extract username // Regular expression pattern to extract username
// from hostname. Must have a single grabbing block. // from hostname. Must have a single grabbing block.
@ -80,8 +83,10 @@ var config = {
cache_ttl: 60000, cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mapnik: { mapnik: {
// The size of the pool of internal mapnik renderers // The size of the pool of internal mapnik backend
// Check the configuration of uv_threadpool_size to use suitable value // This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8, poolSize: 8,
// Metatile is the number of tiles-per-side that are going // Metatile is the number of tiles-per-side that are going
@ -90,6 +95,17 @@ var config = {
// wasted time. // wasted time.
metatile: 2, metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format // Override metatile behaviour depending on the format
formatMetatile: { formatMetatile: {
png: 2, png: 2,
@ -142,6 +158,16 @@ var config = {
type: 'fs', // 'fs' and 'url' supported type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png' src: __dirname + '/../../assets/default-placeholder.png'
} }
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
} }
} }
,millstone: { ,millstone: {
@ -176,6 +202,13 @@ var config = {
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161 unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
} }
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
,httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
}
,varnish: { ,varnish: {
host: '', host: '',
port: null, // the por for the telnet interface where varnish is listening to port: null, // the por for the telnet interface where varnish is listening to

141
lib/cartodb/api/auth_api.js Normal file
View File

@ -0,0 +1,141 @@
var assert = require('assert');
var step = require('step');
/**
*
* @param {PgConnection} pgConnection
* @param metadataBackend
* @param {MapStore} mapStore
* @param {TemplateMaps} templateMaps
* @constructor
* @type {AuthApi}
*/
function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.mapStore = mapStore;
this.templateMaps = templateMaps;
}
module.exports = AuthApi;
// Check if a request is authorized by a signer
//
// @param req express request object
// @param callback function(err, signed_by) signed_by will be
// null if the request is not signed by anyone
// or will be a string cartodb username otherwise.
//
AuthApi.prototype.authorizedBySigner = function(req, callback) {
if ( ! req.params.token || ! req.params.signer ) {
return callback(null, false); // no signer requested
}
var self = this;
var layergroup_id = req.params.token;
var auth_token = req.params.auth_token;
this.mapStore.load(layergroup_id, function(err, mapConfig) {
if (err) {
return callback(err);
}
var authorized = self.templateMaps.isAuthorized(mapConfig.obj().template, auth_token);
return callback(null, authorized);
});
};
// Check if a request is authorized by api_key
//
// @param user
// @param req express request object
// @param callback function(err, authorized)
// NOTE: authorized is expected to be 0 or 1 (integer)
//
AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) {
var givenKey = req.query.api_key || req.query.map_key;
if ( ! givenKey && req.body ) {
// check also in request body
givenKey = req.body.api_key || req.body.map_key;
}
if ( ! givenKey ) {
return callback(null, 0); // no api key, no authorization...
}
var self = this;
step(
function () {
self.metadataBackend.getUserMapKey(user, this);
},
function checkApiKey(err, val){
assert.ifError(err);
return val && givenKey === val;
},
function finish(err, authorized) {
callback(err, authorized);
}
);
};
/**
* Check access authorization
*
* @param req - standard req object. Importantly contains table and host information
* @param callback function(err, allowed) is access allowed not?
*/
AuthApi.prototype.authorize = function(req, callback) {
var self = this;
var user = req.context.user;
step(
function () {
self.authorizedByAPIKey(user, req, this);
},
function checkApiKey(err, authorized){
if (req.profiler) {
req.profiler.done('authorizedByAPIKey');
}
assert.ifError(err);
// if not authorized by api_key, continue
if (!authorized) {
// not authorized by api_key, check if authorized by signer
return self.authorizedBySigner(req, this);
}
// authorized by api key, login as the given username and stop
self.pgConnection.setDBAuth(user, req.params, function(err) {
callback(err, true); // authorized (or error)
});
},
function checkSignAuthorized(err, authorized) {
if (err) {
return callback(err);
}
if ( ! authorized ) {
// request not authorized by signer.
// if no signer name was given, let dbparams and
// PostgreSQL do the rest.
//
if ( ! req.params.signer ) {
return callback(null, true); // authorized so far
}
// if signer name was given, return no authorization
return callback(null, false);
}
self.pgConnection.setDBAuth(user, req.params, function(err) {
if (req.profiler) {
req.profiler.done('setDBAuth');
}
callback(err, true); // authorized (or error)
});
}
);
};

45
lib/cartodb/api/query_tables_api.js
View File

@ -14,7 +14,7 @@ module.exports = QueryTablesApi;
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) { QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)'; var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback); this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback);
}; };
@ -25,9 +25,9 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
callback(new Error('could not fetch source tables: ' + msg)); callback(new Error('could not fetch source tables: ' + msg));
return; return;
} }
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1]; // This is an Array, so no need to split into parts
tableNames = tableNames ? tableNames.split(',') : []; var tableNames = rows[0].cdb_querytablestext;
callback(null, tableNames); callback(null, tableNames);
} }
@ -35,7 +35,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
var query = [ var query = [
'WITH querytables AS (', 'WITH querytables AS (',
'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames', 'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
')', ')',
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max', 'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m', 'FROM CDB_TableMetadata m',
@ -48,14 +48,14 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
if (err || rows.length === 0) { if (err || rows.length === 0) {
var msg = err.message ? err.message : err; var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables and last updated time: ' + msg)); callback(new Error('could not fetch affected tables or last updated time: ' + msg));
return; return;
} }
var result = rows[0]; var result = rows[0];
var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1]; // This is an Array, so no need to split into parts
tableNames = tableNames ? tableNames.split(',') : []; var tableNames = result.tablenames;
var lastUpdatedTime = result.max || 0; var lastUpdatedTime = result.max || 0;
@ -65,6 +65,35 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
}); });
} }
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, 0);
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
this.pgQueryRunner.run(username, query, handleLastUpdatedTimeRows, callback);
};
function handleLastUpdatedTimeRows(err, rows, callback) {
if (err) {
var msg = err.message ? err.message : err;
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var lastUpdated = 0;
if (rows.length !== 0) {
lastUpdated = rows[0].max || 0;
}
return callback(null, lastUpdated*1000);
}
function prepareSql(sql) { function prepareSql(sql) {
return sql return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')

17
lib/cartodb/backends/pg_connection.js
View File

@ -1,3 +1,4 @@
var assert = require('assert');
var step = require('step'); var step = require('step');
var _ = require('underscore'); var _ = require('underscore');
@ -29,19 +30,21 @@ PgConnection.prototype.setDBAuth = function(username, params, callback) {
self.metadataBackend.getUserId(username, this); self.metadataBackend.getUserId(username, this);
}, },
function(err, user_id) { function(err, user_id) {
if (err) throw err; assert.ifError(err);
user_params.user_id = user_id; user_params.user_id = user_id;
var dbuser = _.template(auth_user, user_params); var dbuser = _.template(auth_user, user_params);
_.extend(params, {dbuser:dbuser}); _.extend(params, {dbuser:dbuser});
// skip looking up user_password if postgres_auth_pass // skip looking up user_password if postgres_auth_pass
// doesn't contain the "user_password" label // doesn't contain the "user_password" label
if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null; if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) {
return null;
}
self.metadataBackend.getUserDBPass(username, this); self.metadataBackend.getUserDBPass(username, this);
}, },
function(err, user_password) { function(err, user_password) {
if (err) throw err; assert.ifError(err);
user_params.user_password = user_password; user_params.user_password = user_password;
if ( auth_pass ) { if ( auth_pass ) {
var dbpass = _.template(auth_pass, user_params); var dbpass = _.template(auth_pass, user_params);
@ -81,12 +84,14 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
self.metadataBackend.getUserDBConnectionParams(dbowner, this); self.metadataBackend.getUserDBConnectionParams(dbowner, this);
}, },
function extendParams(err, dbParams){ function extendParams(err, dbParams){
if (err) throw err; assert.ifError(err);
// we don't want null values or overwrite a non public user // we don't want null values or overwrite a non public user
if (params.dbuser != 'publicuser' || !dbParams.dbuser) { if (params.dbuser !== 'publicuser' || !dbParams.dbuser) {
delete dbParams.dbuser; delete dbParams.dbuser;
} }
if ( dbParams ) _.extend(params, dbParams); if ( dbParams ) {
_.extend(params, dbParams);
}
return null; return null;
}, },
function finish(err) { function finish(err) {

9
lib/cartodb/backends/pg_query_runner.js
View File

@ -1,3 +1,4 @@
var assert = require('assert');
var PSQL = require('cartodb-psql'); var PSQL = require('cartodb-psql');
var step = require('step'); var step = require('step');
@ -18,15 +19,11 @@ PgQueryRunner.prototype.run = function(username, query, queryHandler, callback)
self.pgConnection.setDBAuth(username, params, this); self.pgConnection.setDBAuth(username, params, this);
}, },
function setConn(err) { function setConn(err) {
if (err) { assert.ifError(err);
throw err;
}
self.pgConnection.setDBConn(username, params, this); self.pgConnection.setDBConn(username, params, this);
}, },
function executeQuery(err) { function executeQuery(err) {
if (err) { assert.ifError(err);
throw err;
}
var psql = new PSQL({ var psql = new PSQL({
user: params.dbuser, user: params.dbuser,
pass: params.dbpass, pass: params.dbpass,

51
lib/cartodb/backends/template_maps.js
View File

@ -1,3 +1,4 @@
var assert = require('assert');
var crypto = require('crypto'); var crypto = require('crypto');
var step = require('step'); var step = require('step');
var _ = require('underscore'); var _ = require('underscore');
@ -21,7 +22,9 @@ var util = require('util');
// //
// //
function TemplateMaps(redis_pool, opts) { function TemplateMaps(redis_pool, opts) {
if (!(this instanceof TemplateMaps)) return new TemplateMaps(); if (!(this instanceof TemplateMaps)) {
return new TemplateMaps();
}
EventEmitter.call(this); EventEmitter.call(this);
@ -76,13 +79,15 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
that.redis_pool.acquire(db, this); that.redis_pool.acquire(db, this);
}, },
function executeQuery(err, data) { function executeQuery(err, data) {
if ( err ) throw err; assert.ifError(err);
redisClient = data; redisClient = data;
redisArgs.push(this); redisArgs.push(this);
redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs); redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
}, },
function releaseRedisClient(err, data) { function releaseRedisClient(err, data) {
if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient); if ( ! _.isUndefined(redisClient) ) {
that.redis_pool.release(db, redisClient);
}
callback(err, data); callback(err, data);
} }
); );
@ -92,7 +97,7 @@ var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i; var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
// jshint maxcomplexity:15 // jshint maxcomplexity:15
o._checkInvalidTemplate = function(template) { o._checkInvalidTemplate = function(template) {
if ( template.version != '0.0.1' ) { if ( template.version !== '0.0.1' ) {
return new Error("Unsupported template version " + template.version); return new Error("Unsupported template version " + template.version);
} }
var tplname = template.name; var tplname = template.name;
@ -131,10 +136,12 @@ o._checkInvalidTemplate = function(template) {
case 'open': case 'open':
break; break;
case 'token': case 'token':
if ( ! _.isArray(auth.valid_tokens) ) if ( ! _.isArray(auth.valid_tokens) ) {
return new Error("Invalid 'token' authentication: missing valid_tokens"); return new Error("Invalid 'token' authentication: missing valid_tokens");
if ( ! auth.valid_tokens.length ) }
if ( ! auth.valid_tokens.length ) {
return new Error("Invalid 'token' authentication: no valid_tokens"); return new Error("Invalid 'token' authentication: no valid_tokens");
}
break; break;
default: default:
return new Error("Unsupported authentication method: " + auth.method); return new Error("Unsupported authentication method: " + auth.method);
@ -214,9 +221,7 @@ o.addTemplate = function(owner, template, callback) {
self._redisCmd('HLEN', [ userTemplatesKey ], this); self._redisCmd('HLEN', [ userTemplatesKey ], this);
}, },
function installTemplateIfDoesNotExist(err, numberOfTemplates) { function installTemplateIfDoesNotExist(err, numberOfTemplates) {
if ( err ) { assert.ifError(err);
throw err;
}
if ( limit && numberOfTemplates >= limit ) { if ( limit && numberOfTemplates >= limit ) {
throw new Error("User '" + owner + "' reached limit on number of templates " + throw new Error("User '" + owner + "' reached limit on number of templates " +
"("+ numberOfTemplates + "/" + limit + ")"); "("+ numberOfTemplates + "/" + limit + ")");
@ -224,9 +229,7 @@ o.addTemplate = function(owner, template, callback) {
self._redisCmd('HSETNX', [ userTemplatesKey, templateName, JSON.stringify(template) ], this); self._redisCmd('HSETNX', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
}, },
function validateInstallation(err, wasSet) { function validateInstallation(err, wasSet) {
if ( err ) { assert.ifError(err);
throw err;
}
if ( ! wasSet ) { if ( ! wasSet ) {
throw new Error("Template '" + templateName + "' of user '" + owner + "' already exists"); throw new Error("Template '" + templateName + "' of user '" + owner + "' already exists");
} }
@ -259,9 +262,7 @@ o.delTemplate = function(owner, tpl_id, callback) {
self._redisCmd('HDEL', [ self.key_usr_tpl({ owner:owner }), tpl_id ], this); self._redisCmd('HDEL', [ self.key_usr_tpl({ owner:owner }), tpl_id ], this);
}, },
function handleDeletion(err, deleted) { function handleDeletion(err, deleted) {
if (err) { assert.ifError(err);
throw err;
}
if (!deleted) { if (!deleted) {
throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
} }
@ -306,7 +307,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
var templateName = template.name; var templateName = template.name;
if ( tpl_id != templateName ) { if ( tpl_id !== templateName ) {
return callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + templateName + "')")); return callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + templateName + "')"));
} }
@ -317,18 +318,14 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
self._redisCmd('HGET', [ userTemplatesKey, tpl_id ], this); self._redisCmd('HGET', [ userTemplatesKey, tpl_id ], this);
}, },
function updateTemplate(err, currentTemplate) { function updateTemplate(err, currentTemplate) {
if (err) { assert.ifError(err);
throw err;
}
if (!currentTemplate) { if (!currentTemplate) {
throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
} }
self._redisCmd('HSET', [ userTemplatesKey, templateName, JSON.stringify(template) ], this); self._redisCmd('HSET', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
}, },
function handleTemplateUpdate(err, didSetNewField) { function handleTemplateUpdate(err, didSetNewField) {
if (err) { assert.ifError(err);
throw err;
}
if (didSetNewField) { if (didSetNewField) {
console.warn('New template created on update operation'); console.warn('New template created on update operation');
} }
@ -372,7 +369,7 @@ o.getTemplate = function(owner, tpl_id, callback) {
self._redisCmd('HGET', [ self.key_usr_tpl({owner:owner}), tpl_id ], this); self._redisCmd('HGET', [ self.key_usr_tpl({owner:owner}), tpl_id ], this);
}, },
function parseTemplate(err, tpl_val) { function parseTemplate(err, tpl_val) {
if ( err ) throw err; assert.ifError(err);
return JSON.parse(tpl_val); return JSON.parse(tpl_val);
}, },
function finish(err, tpl) { function finish(err, tpl) {
@ -472,8 +469,12 @@ o.instance = function(template, params) {
var layergroup = JSON.parse(JSON.stringify(template.layergroup)); var layergroup = JSON.parse(JSON.stringify(template.layergroup));
for (var i=0; i<layergroup.layers.length; ++i) { for (var i=0; i<layergroup.layers.length; ++i) {
var lyropt = layergroup.layers[i].options; var lyropt = layergroup.layers[i].options;
if ( lyropt.cartocss ) lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params); if ( lyropt.cartocss ) {
if ( lyropt.sql) lyropt.sql = _replaceVars(lyropt.sql, all_params); lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
}
if ( lyropt.sql) {
lyropt.sql = _replaceVars(lyropt.sql, all_params);
}
// Anything else ? // Anything else ?
} }

24
lib/cartodb/cache/layergroup_affected_tables.js vendored Normal file
View File

@ -0,0 +1,24 @@
var LruCache = require('lru-cache');
function LayergroupAffectedTables() {
// dbname + layergroupId -> affected tables cache
this.cache = new LruCache({ max: 2000 });
}
module.exports = LayergroupAffectedTables;
LayergroupAffectedTables.prototype.hasAffectedTables = function(dbName, layergroupId) {
return this.cache.has(createKey(dbName, layergroupId));
};
LayergroupAffectedTables.prototype.set = function(dbName, layergroupId, affectedTables) {
this.cache.set(createKey(dbName, layergroupId), affectedTables);
};
LayergroupAffectedTables.prototype.get = function(dbName, layergroupId) {
return this.cache.get(createKey(dbName, layergroupId));
};
function createKey(dbName, layergroupId) {
return dbName + ':' + layergroupId;
}

24
lib/cartodb/cache/model/database_tables_entry.js vendored Normal file
View File

@ -0,0 +1,24 @@
var crypto = require('crypto');
function DatabaseTables(dbName, tableNames) {
this.namespace = 't';
this.dbName = dbName;
this.tableNames = tableNames;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tableNames.map(function(tableName) {
return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
}.bind(this));
};
DatabaseTables.prototype.getCacheChannel = function() {
return this.dbName + ':' + this.tableNames.join(',');
};
function shortHashKey(target) {
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
}

62
lib/cartodb/cache/named_map_provider_cache.js vendored Normal file
View File

@ -0,0 +1,62 @@
var _ = require('underscore');
var dot = require('dot');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
var templateName = require('../backends/template_maps').templateName;
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.providerCache = new LruCache({ max: 2000 });
}
module.exports = NamedMapProviderCache;
NamedMapProviderCache.prototype.get = function(user, templateId, config, authToken, params) {
var namedMapKey = createNamedMapKey(user, templateId);
var namedMapProviders = this.providerCache.get(namedMapKey) || {};
var providerKey = createProviderKey(config, authToken, params);
if (!namedMapProviders.hasOwnProperty(providerKey)) {
namedMapProviders[providerKey] = new NamedMapMapConfigProvider(
this.templateMaps,
this.pgConnection,
this.userLimitsApi,
this.queryTablesApi,
user,
templateId,
config,
authToken,
params
);
this.providerCache.set(namedMapKey, namedMapProviders);
}
return namedMapProviders[providerKey];
};
NamedMapProviderCache.prototype.invalidate = function(user, templateId) {
this.providerCache.del(createNamedMapKey(user, templateId));
};
function createNamedMapKey(user, templateId) {
return user + ':' + templateName(templateId);
}
var providerKey = '{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}';
var providerKeyTpl = dot.template(providerKey);
function createProviderKey(config, authToken, params) {
var tplValues = _.defaults({}, params, {
authToken: authToken || '',
configHash: NamedMapMapConfigProvider.configHash(config),
layer: '',
format: '',
scale_factor: 1
});
return providerKeyTpl(tplValues);
}

14
lib/cartodb/cache/surrogate_keys_cache.js vendored
View File

@ -16,9 +16,21 @@ module.exports = SurrogateKeysCache;
* @param cacheObject should respond to `key() -> String` method * @param cacheObject should respond to `key() -> String` method
*/ */
SurrogateKeysCache.prototype.tag = function(response, cacheObject) { SurrogateKeysCache.prototype.tag = function(response, cacheObject) {
response.header('Surrogate-Key', cacheObject.key()); var newKey = cacheObject.key();
response.header('Surrogate-Key', appendSurrogateKey(
response.header('Surrogate-Key'),
Array.isArray(newKey) ? cacheObject.key().join(' ') : newKey
));
}; };
function appendSurrogateKey(currentKey, newKey) {
if (!!currentKey) {
newKey = currentKey + ' ' + newKey;
}
return newKey;
}
/** /**
* @param cacheObject should respond to `key() -> String` method * @param cacheObject should respond to `key() -> String` method
* @param {Function} callback * @param {Function} callback

107
lib/cartodb/controllers/layergroup.js
View File

@ -4,6 +4,7 @@ var step = require('step');
var cors = require('../middleware/cors'); var cors = require('../middleware/cors');
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
/** /**
* @param app * @param app
@ -11,16 +12,23 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'
* @param {TileBackend} tileBackend * @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend * @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend * @param {AttributesBackend} attributesBackend
* @param {{UserLimitsApi}} userLimitsApi * @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor * @constructor
*/ */
function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi) { function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, surrogateKeysCache,
userLimitsApi, queryTablesApi, layergroupAffectedTables) {
this.app = app; this.app = app;
this.mapStore = mapStore; this.mapStore = mapStore;
this.tileBackend = tileBackend; this.tileBackend = tileBackend;
this.previewBackend = previewBackend; this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend; this.attributesBackend = attributesBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi; this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.layergroupAffectedTables = layergroupAffectedTables;
} }
module.exports = LayergroupController; module.exports = LayergroupController;
@ -62,7 +70,7 @@ LayergroupController.prototype.attributes = function(req, res) {
var statusCode = self.app.findStatusCode(err); var statusCode = self.app.findStatusCode(err);
self.app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err); self.app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err);
} else { } else {
self.app.sendResponse(res, [tile, 200]); self.sendResponse(req, res, [tile, 200]);
} }
} }
); );
@ -87,8 +95,6 @@ LayergroupController.prototype.layer = function(req, res, next) {
LayergroupController.prototype.tileOrLayer = function (req, res) { LayergroupController.prototype.tileOrLayer = function (req, res) {
var self = this; var self = this;
console.log(req.context.user);
step( step(
function mapController$prepareParams() { function mapController$prepareParams() {
self.app.req2params(req, this); self.app.req2params(req, this);
@ -149,7 +155,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.error');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
} else { } else {
this.app.sendWithHeaders(res, tile, 200, headers); this.sendResponse(req, res, [tile, headers, 200]);
global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.success');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
} }
@ -206,8 +212,95 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
self.app.sendError(res, {errors: ['' + err] }, self.app.findStatusCode(err), 'STATIC_MAP', err); self.app.sendError(res, {errors: ['' + err] }, self.app.findStatusCode(err), 'STATIC_MAP', err);
} else { } else {
res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format); res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format);
self.app.sendResponse(res, [image, 200]); self.sendResponse(req, res, [image, 200]);
} }
} }
); );
}; };
LayergroupController.prototype.sendResponse = function(req, res, args) {
var self = this;
res.header('Cache-Control', 'public,max-age=31536000');
// Set Last-Modified header
var lastUpdated;
if (req.params.cache_buster) {
// Assuming cache_buster is a timestamp
lastUpdated = new Date(parseInt(req.params.cache_buster));
} else {
lastUpdated = new Date();
}
res.header('Last-Modified', lastUpdated.toUTCString());
var dbName = req.params.dbname;
step(
function getAffectedTables() {
self.getAffectedTables(req.context.user, dbName, req.params.token, this);
},
function sendResponse(err, affectedTables) {
req.profiler.done('affectedTables');
if (err) {
console.log('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
}
self.app.sendResponse(res, args);
}
);
};
LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) {
return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId));
}
var self = this;
step(
function extractSQL() {
step(
function loadFromStore() {
self.mapStore.load(layergroupId, this);
},
function getSQL(err, mapConfig) {
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
return tableNames;
},
function finish(err, affectedTables) {
callback(err, affectedTables);
}
);
};

60
lib/cartodb/controllers/map.js
View File

@ -9,6 +9,7 @@ var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource; var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
@ -22,11 +23,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye
* @param metadataBackend * @param metadataBackend
* @param {QueryTablesApi} queryTablesApi * @param {QueryTablesApi} queryTablesApi
* @param {SurrogateKeysCache} surrogateKeysCache * @param {SurrogateKeysCache} surrogateKeysCache
* @param {{UserLimitsApi}} userLimitsApi * @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor * @constructor
*/ */
function MapController(app, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi, function MapController(app, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi,
surrogateKeysCache, userLimitsApi) { surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
this.app = app; this.app = app;
this.pgConnection = pgConnection; this.pgConnection = pgConnection;
this.templateMaps = templateMaps; this.templateMaps = templateMaps;
@ -35,6 +37,8 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack
this.queryTablesApi = queryTablesApi; this.queryTablesApi = queryTablesApi;
this.surrogateKeysCache = surrogateKeysCache; this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi; this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTables = layergroupAffectedTables;
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
} }
@ -147,13 +151,14 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
}, },
function afterLayergroupCreate(err, layergroup) { function afterLayergroupCreate(err, layergroup) {
assert.ifError(err); assert.ifError(err);
self.afterLayergroupCreate(req, mapConfig, layergroup, this); self.afterLayergroupCreate(req, res, mapConfig, layergroup, this);
}, },
function finish(err, layergroup) { function finish(err, layergroup) {
if (err) { if (err) {
var statusCode = self.app.findStatusCode(err); var statusCode = self.app.findStatusCode(err);
self.app.sendError(res, { errors: [ err.message ] }, statusCode, 'ANONYMOUS LAYERGROUP', err); self.app.sendError(res, { errors: [ err.message ] }, statusCode, 'ANONYMOUS LAYERGROUP', err);
} else { } else {
res.header('X-Layergroup-Id', layergroup.layergroupid);
self.app.sendResponse(res, [layergroup, 200]); self.app.sendResponse(res, [layergroup, 200]);
} }
} }
@ -169,6 +174,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
var mapConfig; var mapConfig;
step( step(
function setupParams(){
self.app.req2params(req, this);
},
function getTemplateParams() { function getTemplateParams() {
prepareParamsFn(this); prepareParamsFn(this);
}, },
@ -178,6 +186,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
self.templateMaps, self.templateMaps,
self.pgConnection, self.pgConnection,
self.userLimitsApi, self.userLimitsApi,
self.queryTablesApi,
cdbuser, cdbuser,
req.params.template_id, req.params.template_id,
templateParams, templateParams,
@ -197,7 +206,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
}, },
function afterLayergroupCreate(err, layergroup) { function afterLayergroupCreate(err, layergroup) {
assert.ifError(err); assert.ifError(err);
self.afterLayergroupCreate(req, mapConfig, layergroup, this); self.afterLayergroupCreate(req, res, mapConfig, layergroup, this);
}, },
function finishTemplateInstantiation(err, layergroup) { function finishTemplateInstantiation(err, layergroup) {
if (err) { if (err) {
@ -217,7 +226,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
}; };
MapController.prototype.afterLayergroupCreate = function(req, mapconfig, layergroup, callback) { MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, callback) {
var self = this; var self = this;
var username = req.context.user; var username = req.context.user;
@ -258,34 +267,47 @@ MapController.prototype.afterLayergroupCreate = function(req, mapconfig, layergr
}).join(';'); }).join(';');
var dbName = req.params.dbname; var dbName = req.params.dbname;
var cacheKey = dbName + ':' + layergroup.layergroupid; var layergroupId = layergroup.layergroupid;
step( step(
function getAffectedTablesAndLastUpdatedTime() { function checkCachedAffectedTables() {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
},
function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
assert.ifError(err);
if (hasCache) {
var next = this;
var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
if (err) {
return next(err);
}
return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
});
} else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
}
}, },
function handleAffectedTablesAndLastUpdatedTime(err, result) { function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) { if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated'); req.profiler.done('queryTablesAndLastUpdated');
} }
assert.ifError(err); assert.ifError(err);
var cacheChannel = self.app.buildCacheChannel(dbName, result.affectedTables); self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
self.app.channelCache[cacheKey] = cacheChannel;
// last update for layergroup cache buster // last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString(); layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
var res = req.res; if (req.method === 'GET') {
if (res) { var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
if (req.method === 'GET') { var ttl = global.environment.varnish.layergroupTtl || 86400;
var ttl = global.environment.varnish.layergroupTtl || 86400; res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); res.header('Last-Modified', (new Date()).toUTCString());
res.header('Last-Modified', (new Date()).toUTCString()); res.header('X-Cache-Channel', tableCacheEntry.getCacheChannel());
res.header('X-Cache-Channel', cacheChannel); if (result.affectedTables && result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tableCacheEntry);
} }
res.header('X-Layergroup-Id', layergroup.layergroupid);
} }
return null; return null;

70
lib/cartodb/controllers/named_maps.js
View File

@ -4,18 +4,16 @@ var _ = require('underscore');
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var cors = require('../middleware/cors'); var cors = require('../middleware/cors');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var TablesCacheEntry = require('../cache/model/database_tables_entry');
function NamedMapsController(app, pgConnection, templateMaps, tileBackend, previewBackend, surrogateKeysCache, function NamedMapsController(app, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache,
tablesExtentApi, userLimitsApi) { tablesExtentApi) {
this.app = app; this.app = app;
this.pgConnection = pgConnection; this.namedMapProviderCache = namedMapProviderCache;
this.templateMaps = templateMaps;
this.tileBackend = tileBackend; this.tileBackend = tileBackend;
this.previewBackend = previewBackend; this.previewBackend = previewBackend;
this.surrogateKeysCache = surrogateKeysCache; this.surrogateKeysCache = surrogateKeysCache;
this.tablesExtentApi = tablesExtentApi; this.tablesExtentApi = tablesExtentApi;
this.userLimitsApi = userLimitsApi;
} }
module.exports = NamedMapsController; module.exports = NamedMapsController;
@ -27,6 +25,46 @@ NamedMapsController.prototype.register = function(app) {
); );
}; };
NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) {
this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName()));
res.header('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
res.header('Cache-Control', 'public,max-age=7200,must-revalidate');
var self = this;
var dbName = req.params.dbname;
step(
function getAffectedTablesAndLastUpdatedTime() {
namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
},
function sendResponse(err, result) {
req.profiler.done('affectedTables');
if (err) {
console.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.affectedTables) {
// we increase cache control as we can invalidate it
res.header('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.lastUpdatedTime);
} else {
lastModifiedDate = new Date();
}
res.header('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);
}
}
self.app.sendResponse(res, [resource, 200]);
}
);
};
NamedMapsController.prototype.tile = function(req, res) { NamedMapsController.prototype.tile = function(req, res) {
var self = this; var self = this;
@ -38,10 +76,7 @@ NamedMapsController.prototype.tile = function(req, res) {
self.app.req2params(req, this); self.app.req2params(req, this);
}, },
function getTile() { function getTile() {
namedMapProvider = new NamedMapMapConfigProvider( namedMapProvider = self.namedMapProviderCache.get(
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
cdbUser, cdbUser,
req.params.template_id, req.params.template_id,
req.query.config, req.query.config,
@ -60,10 +95,7 @@ NamedMapsController.prototype.tile = function(req, res) {
} }
self.app.sendError(res, err, self.app.findStatusCode(err), 'NAMED_MAP_TILE', err); self.app.sendError(res, err, self.app.findStatusCode(err), 'NAMED_MAP_TILE', err);
} else { } else {
self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName())); self.sendResponse(req, res, tile, headers, namedMapProvider);
res.setHeader('Content-Type', headers['Content-Type']);
res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate');
self.app.sendWithHeaders(res, tile, 200, headers);
} }
} }
); );
@ -85,10 +117,7 @@ NamedMapsController.prototype.staticMap = function(req, res) {
}, },
function getTemplate(err) { function getTemplate(err) {
assert.ifError(err); assert.ifError(err);
namedMapProvider = new NamedMapMapConfigProvider( namedMapProvider = self.namedMapProviderCache.get(
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
cdbUser, cdbUser,
req.params.template_id, req.params.template_id,
req.query.config, req.query.config,
@ -158,10 +187,7 @@ NamedMapsController.prototype.staticMap = function(req, res) {
} }
self.app.sendError(res, err, self.app.findStatusCode(err), 'STATIC_VIZ_MAP', err); self.app.sendError(res, err, self.app.findStatusCode(err), 'STATIC_VIZ_MAP', err);
} else { } else {
self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName())); self.sendResponse(req, res, image, headers, namedMapProvider);
res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format);
res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate');
self.app.sendResponse(res, [image, 200]);
} }
} }
); );

23
lib/cartodb/controllers/named_maps_admin.js
View File

@ -5,9 +5,16 @@ var templateName = require('../backends/template_maps').templateName;
var cors = require('../middleware/cors'); var cors = require('../middleware/cors');
function NamedMapsAdminController(app, templateMaps) { /**
* @param app
* @param {TemplateMaps} templateMaps
* @param {AuthApi} authApi
* @constructor
*/
function NamedMapsAdminController(app, templateMaps, authApi) {
this.app = app; this.app = app;
this.templateMaps = templateMaps; this.templateMaps = templateMaps;
this.authApi = authApi;
} }
module.exports = NamedMapsAdminController; module.exports = NamedMapsAdminController;
@ -28,7 +35,7 @@ NamedMapsAdminController.prototype.create = function(req, res) {
step( step(
function checkPerms(){ function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this); self.authApi.authorizedByAPIKey(cdbuser, req, this);
}, },
function addTemplate(err, authenticated) { function addTemplate(err, authenticated) {
assert.ifError(err); assert.ifError(err);
@ -53,7 +60,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
var tpl_id; var tpl_id;
step( step(
function checkPerms(){ function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this); self.authApi.authorizedByAPIKey(cdbuser, req, this);
}, },
function updateTemplate(err, authenticated) { function updateTemplate(err, authenticated) {
assert.ifError(err); assert.ifError(err);
@ -84,7 +91,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
var tpl_id; var tpl_id;
step( step(
function checkPerms(){ function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this); self.authApi.authorizedByAPIKey(cdbuser, req, this);
}, },
function getTemplate(err, authenticated) { function getTemplate(err, authenticated) {
assert.ifError(err); assert.ifError(err);
@ -94,7 +101,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
self.templateMaps.getTemplate(cdbuser, tpl_id, this); self.templateMaps.getTemplate(cdbuser, tpl_id, this);
}, },
function prepareResponse(err, tpl_val) { function prepareResponse(err, tpl_val) {
if ( err ) throw err; assert.ifError(err);
if ( ! tpl_val ) { if ( ! tpl_val ) {
err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'"); err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
err.http_status = 404; err.http_status = 404;
@ -120,7 +127,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
var tpl_id; var tpl_id;
step( step(
function checkPerms(){ function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this); self.authApi.authorizedByAPIKey(cdbuser, req, this);
}, },
function deleteTemplate(err, authenticated) { function deleteTemplate(err, authenticated) {
assert.ifError(err); assert.ifError(err);
@ -130,7 +137,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
self.templateMaps.delTemplate(cdbuser, tpl_id, this); self.templateMaps.delTemplate(cdbuser, tpl_id, this);
}, },
function prepareResponse(err/*, tpl_val*/){ function prepareResponse(err/*, tpl_val*/){
if ( err ) throw err; assert.ifError(err);
return { status: 'ok' }; return { status: 'ok' };
}, },
finishFn(self.app, res, 'DELETE TEMPLATE', ['', 204]) finishFn(self.app, res, 'DELETE TEMPLATE', ['', 204])
@ -147,7 +154,7 @@ NamedMapsAdminController.prototype.list = function(req, res) {
step( step(
function checkPerms(){ function checkPerms(){
self.app.authorizedByAPIKey(cdbuser, req, this); self.authApi.authorizedByAPIKey(cdbuser, req, this);
}, },
function listTemplates(err, authenticated) { function listTemplates(err, authenticated) {
assert.ifError(err); assert.ifError(err);

7
lib/cartodb/controllers/server_info.js
View File

@ -15,7 +15,7 @@ var versions = {
function ServerInfoController() { function ServerInfoController() {
this.healthConfig = global.environment.health || {}; this.healthConfig = global.environment.health || {};
this.healthCheck = new HealthCheck(); this.healthCheck = new HealthCheck(global.environment.disabled_file);
} }
module.exports = ServerInfoController; module.exports = ServerInfoController;
@ -37,13 +37,12 @@ ServerInfoController.prototype.version = function(req, res) {
ServerInfoController.prototype.health = function(req, res) { ServerInfoController.prototype.health = function(req, res) {
if (!!this.healthConfig.enabled) { if (!!this.healthConfig.enabled) {
var startTime = Date.now(); var startTime = Date.now();
this.healthCheck.check(this.healthConfig, function(err, result) { this.healthCheck.check(function(err) {
var ok = !err; var ok = !err;
var response = { var response = {
enabled: true, enabled: true,
ok: ok, ok: ok,
elapsed: Date.now() - startTime, elapsed: Date.now() - startTime
result: result
}; };
if (err) { if (err) {
response.err = err.message; response.err = err.message;

45
lib/cartodb/models/mapconfig/named_map_provider.js
View File

@ -10,20 +10,25 @@ var templateName = require('../../backends/template_maps').templateName;
* @constructor * @constructor
* @type {NamedMapMapConfigProvider} * @type {NamedMapMapConfigProvider}
*/ */
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, owner, templateId, config, authToken, function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi,
params) { owner, templateId, config, authToken, params) {
this.templateMaps = templateMaps; this.templateMaps = templateMaps;
this.pgConnection = pgConnection; this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi; this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.owner = owner; this.owner = owner;
this.templateName = templateName(templateId); this.templateName = templateName(templateId);
this.config = config; this.config = config;
this.authToken = authToken; this.authToken = authToken;
this.params = params; this.params = params;
this.cacheBuster = Date.now();
// use template after call to mapConfig // use template after call to mapConfig
this.template = null; this.template = null;
this.affectedTablesAndLastUpdate = null;
// providing // providing
this.err = null; this.err = null;
this.mapConfig = null; this.mapConfig = null;
@ -144,7 +149,7 @@ NamedMapMapConfigProvider.prototype.getKey = function() {
}; };
NamedMapMapConfigProvider.prototype.getCacheBuster = function() { NamedMapMapConfigProvider.prototype.getCacheBuster = function() {
return 0; return this.cacheBuster;
}; };
NamedMapMapConfigProvider.prototype.filter = function(key) { NamedMapMapConfigProvider.prototype.filter = function(key) {
@ -153,7 +158,7 @@ NamedMapMapConfigProvider.prototype.filter = function(key) {
}; };
// Configure bases for cache keys suitable for string interpolation // Configure bases for cache keys suitable for string interpolation
var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}'; var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}';
var rendererKey = baseKey + ':{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}'; var rendererKey = baseKey + ':{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}';
var baseKeyTpl = dot.template(baseKey); var baseKeyTpl = dot.template(baseKey);
@ -179,6 +184,8 @@ function configHash(config) {
return crypto.createHash('md5').update(JSON.stringify(config)).digest('hex').substring(0,8); return crypto.createHash('md5').update(JSON.stringify(config)).digest('hex').substring(0,8);
} }
module.exports.configHash = configHash;
NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, callback) { NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, callback) {
var self = this; var self = this;
step( step(
@ -186,7 +193,7 @@ NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, call
self.pgConnection.setDBAuth(cdbuser, params, this); self.pgConnection.setDBAuth(cdbuser, params, this);
}, },
function setConn(err) { function setConn(err) {
if ( err ) throw err; assert.ifError(err);
self.pgConnection.setDBConn(cdbuser, params, this); self.pgConnection.setDBConn(cdbuser, params, this);
}, },
function finish(err) { function finish(err) {
@ -198,3 +205,31 @@ NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, call
NamedMapMapConfigProvider.prototype.getTemplateName = function() { NamedMapMapConfigProvider.prototype.getTemplateName = function() {
return this.templateName; return this.templateName;
}; };
NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) {
var self = this;
if (this.affectedTablesAndLastUpdate !== null) {
return callback(null, this.affectedTablesAndLastUpdate);
}
step(
function getMapConfig() {
self.getMapConfig(this);
},
function getSql(err, mapConfig) {
assert.ifError(err);
return mapConfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
},
function getAffectedTables(err, sql) {
assert.ifError(err);
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;
return callback(err, result);
}
);
};

21
lib/cartodb/monitoring/health_check.js
View File

@ -1,29 +1,20 @@
var fs = require('fs'); var fs = require('fs');
var step = require('step'); var step = require('step');
function HealthCheck() { function HealthCheck(disableFile) {
this.disableFile = disableFile;
} }
module.exports = HealthCheck; module.exports = HealthCheck;
HealthCheck.prototype.check = function(config, callback) { HealthCheck.prototype.check = function(callback) {
var result = { var self = this;
redis: {
ok: false
},
mapnik: {
ok: false
},
tile: {
ok: false
}
};
step( step(
function getManualDisable() { function getManualDisable() {
fs.readFile(global.environment.disabled_file, this); fs.readFile(self.disableFile, this);
}, },
function handleDisabledFile(err, data) { function handleDisabledFile(err, data) {
var next = this; var next = this;
@ -37,7 +28,7 @@ HealthCheck.prototype.check = function(config, callback) {
} }
}, },
function handleResult(err) { function handleResult(err) {
callback(err, result); return callback(err);
} }
); );
}; };

411
lib/cartodb/server.js
View File

@ -19,6 +19,9 @@ var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js'); var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api'); var QueryTablesApi = require('./api/query_tables_api');
var UserLimitsApi = require('./api/user_limits_api'); var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
var NamedMapProviderCache = require('./cache/named_map_provider_cache');
var PgQueryRunner = require('./backends/pg_query_runner'); var PgQueryRunner = require('./backends/pg_query_runner');
var PgConnection = require('./backends/pg_connection'); var PgConnection = require('./backends/pg_connection');
@ -73,12 +76,6 @@ module.exports = function(serverOptions) {
max_user_templates: global.environment.maxUserTemplates max_user_templates: global.environment.maxUserTemplates
}); });
// This is for Templated maps
//
// "named" is the official, "template" is for backward compatibility up to 1.6.x
//
var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
var surrogateKeysCacheBackends = []; var surrogateKeysCacheBackends = [];
if (serverOptions.varnish_purge_enabled) { if (serverOptions.varnish_purge_enabled) {
@ -131,7 +128,6 @@ module.exports = function(serverOptions) {
pool: redisPool, pool: redisPool,
expire_time: serverOptions.grainstore.default_layergroup_ttl expire_time: serverOptions.grainstore.default_layergroup_ttl
}); });
app.mapStore = mapStore;
var onTileErrorStrategy; var onTileErrorStrategy;
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
@ -167,6 +163,16 @@ module.exports = function(serverOptions) {
var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
});
var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);
app.findStatusCode = function(err) { app.findStatusCode = function(err) {
var statusCode; var statusCode;
if ( err.http_status ) { if ( err.http_status ) {
@ -195,7 +201,10 @@ module.exports = function(serverOptions) {
tileBackend, tileBackend,
previewBackend, previewBackend,
attributesBackend, attributesBackend,
userLimitsApi surrogateKeysCache,
userLimitsApi,
queryTablesApi,
layergroupAffectedTablesCache
).register(app); ).register(app);
new controller.Map( new controller.Map(
@ -206,21 +215,20 @@ module.exports = function(serverOptions) {
metadataBackend, metadataBackend,
queryTablesApi, queryTablesApi,
surrogateKeysCache, surrogateKeysCache,
userLimitsApi userLimitsApi,
layergroupAffectedTablesCache
).register(app); ).register(app);
new controller.NamedMaps( new controller.NamedMaps(
app, app,
pgConnection, namedMapProviderCache,
templateMaps,
tileBackend, tileBackend,
previewBackend, previewBackend,
surrogateKeysCache, surrogateKeysCache,
tablesExtentApi, tablesExtentApi
userLimitsApi
).register(app); ).register(app);
new controller.NamedMapsAdmin(app, templateMaps).register(app); new controller.NamedMapsAdmin(app, templateMaps, authApi).register(app);
new controller.ServerInfo().register(app); new controller.ServerInfo().register(app);
@ -241,100 +249,33 @@ module.exports = function(serverOptions) {
} }
}); });
// GET routes for which we don't want to request any caching.
// POST/PUT/DELETE requests are never cached anyway.
var noCacheGETRoutes = [
'/',
'/version',
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
serverOptions.base_url_mapconfig,
serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
template_baseurl,
template_baseurl + '/:template_id',
template_baseurl + '/:template_id/jsonp'
];
app.sendResponse = function(res, args) { app.sendResponse = function(res, args) {
var that = this; var req = res.req;
var statusCode; if (global.environment && global.environment.api_hostname) {
if ( res._windshaftStatusCode ) { res.header('X-Served-By-Host', global.environment.api_hostname);
// Added by our override of sendError }
statusCode = res._windshaftStatusCode;
} else { if (req && req.params && req.params.dbhost) {
if ( args.length > 2 ) statusCode = args[2]; res.header('X-Served-By-DB-Host', req.params.dbhost);
else { }
statusCode = args[1] || 200;
if ( req && req.profiler ) {
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
// res.send(body|status[, headers|status[, status]])
res.send.apply(res, args);
if ( req && req.profiler ) {
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
console.error("error sending profiling stats: " + err);
} }
} }
var req = res.req;
step (
function addCacheChannel() {
if ( ! req ) {
// having no associated request can happen when
// using fake response objects for testing layergroup
// creation
return false;
}
if ( ! req.params ) {
// service requests (/version, /)
// have no need for an X-Cache-Channel
return false;
}
if ( statusCode != 200 ) {
// We do not want to cache
// unsuccessful responses
return false;
}
if ( _.contains(noCacheGETRoutes, req.route.path) ) {
//console.log("Skipping cache channel in route:\n" + req.route.path);
return false;
}
//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" +
// mapCreateRoutes.join("\n"));
app.addCacheChannel(that, req, this);
},
function sendResponse(err/*, added*/) {
if ( err ) console.log(err + err.stack);
// When using custom results from tryFetch* methods,
// there is no "req" link in the result object.
// In those cases we don't want to send stats now
// as they will be sent at the real end of request
var req = res.req;
if (global.environment && global.environment.api_hostname) {
res.header('X-Served-By-Host', global.environment.api_hostname);
}
if (req && req.params && req.params.dbhost) {
res.header('X-Served-By-DB-Host', req.params.dbhost);
}
if ( req && req.profiler ) {
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
res.send.apply(res, args);
if ( req && req.profiler ) {
try {
// May throw due to dns, see
// See http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();
} catch (err) {
console.error("error sending profiling stats: " + err);
}
}
return null;
},
function finish(err) {
if ( err ) console.log(err + err.stack);
}
);
};
app.sendWithHeaders = function(res, what, status, headers) {
app.sendResponse(res, [what, headers, status]);
}; };
app.sendError = function(res, err, statusCode, label, tolog) { app.sendError = function(res, err, statusCode, label, tolog) {
@ -453,9 +394,9 @@ module.exports = function(serverOptions) {
step( step(
function getPrivacy(){ function getPrivacy(){
app.authorize(req, this); authApi.authorize(req, this);
}, },
function gatekeep(err, authorized){ function validateAuthorization(err, authorized) {
if (req.profiler) { if (req.profiler) {
req.profiler.done('authorize'); req.profiler.done('authorize');
} }
@ -472,7 +413,9 @@ module.exports = function(serverOptions) {
pgConnection.setDBConn(user, req.params, this); pgConnection.setDBConn(user, req.params, this);
}, },
function finishSetup(err) { function finishSetup(err) {
if ( err ) { callback(err, req); return; } if ( err ) {
return callback(err, req);
}
// Add default database connection parameters // Add default database connection parameters
// if none given // if none given
@ -488,260 +431,6 @@ module.exports = function(serverOptions) {
); );
}; };
// TODO: review lifetime of elements of this cache
// NOTE: by-token indices should only be dropped when
// the corresponding layegroup is dropped, because
// we have no SQL after layer creation.
app.channelCache = {};
app.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};
app.generateCacheChannel = function(app, req, callback){
// Build channelCache key
var dbName = req.params.dbname;
var cacheKey = [ dbName, req.params.token ].join(':');
// no token means no tables associated
if (!req.params.token) {
return callback(null, this.buildCacheChannel(dbName, []));
}
step(
function checkCached() {
if ( app.channelCache.hasOwnProperty(cacheKey) ) {
return callback(null, app.channelCache[cacheKey]);
}
return null;
},
function extractSQL(err) {
assert.ifError(err);
// TODO: cached cache channel for token-based access should
// be constructed at renderer cache creation time
// See http://github.com/CartoDB/Windshaft-cartodb/issues/152
if ( ! app.mapStore ) {
throw new Error('missing channel cache for token ' + req.params.token);
}
var mapStore = app.mapStore;
step(
function loadFromStore() {
mapStore.load(req.params.token, this);
},
function getSQL(err, mapConfig) {
if (req.profiler) {
req.profiler.done('mapStore_load');
}
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
assert.ifError(err);
if (req.profiler) {
req.profiler.done('affectedTables');
}
var cacheChannel = app.buildCacheChannel(dbName,tableNames);
app.channelCache[cacheKey] = cacheChannel;
return cacheChannel;
},
function finish(err, cacheChannel) {
callback(err, cacheChannel);
}
);
};
// Set the cache chanel info to invalidate the cache on the frontend server
//
// @param req The request object.
// The function will have no effect unless req.res exists.
// It is expected that req.params contains 'table' and 'dbname'
//
// @param cb function(err, channel) will be called when ready.
// the channel parameter will be null if nothing was added
//
app.addCacheChannel = function(app, req, cb) {
// skip non-GET requests, or requests for which there's no response
if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
if (req.profiler) {
req.profiler.start('addCacheChannel');
}
var res = req.res;
if ( req.params.token ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
var ttl = global.environment.varnish.ttl || 86400;
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
}
// Set Last-Modified header
var lastUpdated;
if ( req.params.cache_buster ) {
// Assuming cache_buster is a timestamp
// FIXME: store lastModified in the cache channel instead
lastUpdated = new Date(parseInt(req.params.cache_buster));
} else {
lastUpdated = new Date();
}
res.header('Last-Modified', lastUpdated.toUTCString());
app.generateCacheChannel(app, req, function(err, channel){
if (req.profiler) {
req.profiler.done('generateCacheChannel');
req.profiler.end();
}
if ( ! err ) {
res.header('X-Cache-Channel', channel);
cb(null, channel);
} else {
console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
// TODO: evaluate if we should bubble up the error instead
cb(null, 'ERROR');
}
});
};
// Check if a request is authorized by a signer
//
// @param req express request object
// @param callback function(err, signed_by) signed_by will be
// null if the request is not signed by anyone
// or will be a string cartodb username otherwise.
//
app.authorizedBySigner = function(req, callback) {
if ( ! req.params.token || ! req.params.signer ) {
return callback(null, false); // no signer requested
}
var layergroup_id = req.params.token;
var auth_token = req.params.auth_token;
mapStore.load(layergroup_id, function(err, mapConfig) {
if (err) {
return callback(err);
}
var authorized = templateMaps.isAuthorized(mapConfig.obj().template, auth_token);
return callback(null, authorized);
});
};
// Check if a request is authorized by api_key
//
// @param user
// @param req express request object
// @param callback function(err, authorized)
// NOTE: authorized is expected to be 0 or 1 (integer)
//
app.authorizedByAPIKey = function(user, req, callback) {
var givenKey = req.query.api_key || req.query.map_key;
if ( ! givenKey && req.body ) {
// check also in request body
givenKey = req.body.api_key || req.body.map_key;
}
if ( ! givenKey ) {
return callback(null, 0); // no api key, no authorization...
}
step(
function () {
metadataBackend.getUserMapKey(user, this);
},
function checkApiKey(err, val){
assert.ifError(err);
return val && givenKey == val;
},
function finish(err, authorized) {
callback(err, authorized);
}
);
};
/**
* Check access authorization
*
* @param req - standard req object. Importantly contains table and host information
* @param callback function(err, allowed) is access allowed not?
*/
app.authorize = function(req, callback) {
var self = this;
var user = req.context.user;
step(
function () {
self.authorizedByAPIKey(user, req, this);
},
function checkApiKey(err, authorized){
if (req.profiler) {
req.profiler.done('authorizedByAPIKey');
}
assert.ifError(err);
// if not authorized by api_key, continue
if (!authorized) {
// not authorized by api_key, check if authorized by signer
return self.authorizedBySigner(req, this);
}
// authorized by api key, login as the given username and stop
pgConnection.setDBAuth(user, req.params, function(err) {
callback(err, true); // authorized (or error)
});
},
function checkSignAuthorized(err, authorized) {
if (err) {
return callback(err);
}
if ( ! authorized ) {
// request not authorized by signer.
// if no signer name was given, let dbparams and
// PostgreSQL do the rest.
//
if ( ! req.params.signer ) {
return callback(null, true); // authorized so far
}
// if signer name was given, return no authorization
return callback(null, false);
}
pgConnection.setDBAuth(user, req.params, function(err) {
if (req.profiler) {
req.profiler.done('setDBAuth');
}
callback(err, true); // authorized (or error)
});
}
);
};
return app; return app;
}; };

1
lib/cartodb/server_options.js
View File

@ -62,6 +62,7 @@ module.exports = {
}, },
renderer: { renderer: {
mapnik: rendererConfig.mapnik, mapnik: rendererConfig.mapnik,
torque: rendererConfig.torque,
http: rendererConfig.http http: rendererConfig.http
}, },
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161 // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161

1048
npm-shrinkwrap.json generated
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@ -1,7 +1,7 @@
{ {
"private": true, "private": true,
"name": "windshaft-cartodb", "name": "windshaft-cartodb",
"version": "2.7.2", "version": "2.12.1",
"description": "A map tile server for CartoDB", "description": "A map tile server for CartoDB",
"keywords": [ "keywords": [
"cartodb" "cartodb"
@ -33,6 +33,7 @@
"cartodb-psql": "~0.4.0", "cartodb-psql": "~0.4.0",
"fastly-purge": "~1.0.0", "fastly-purge": "~1.0.0",
"redis-mpool": "~0.4.0", "redis-mpool": "~0.4.0",
"lru-cache": "2.6.5",
"lzma": "~1.3.7", "lzma": "~1.3.7",
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb" "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
}, },

2
test/acceptance/health_check.js
View File

@ -60,7 +60,7 @@ describe('health checks', function () {
callback(null, "Maintenance"); callback(null, "Maintenance");
}; };
healthCheck.check(null, function(err/*, result*/) { healthCheck.check(function(err) {
assert.equal(err.message, "Maintenance"); assert.equal(err.message, "Maintenance");
assert.equal(err.http_status, 503); assert.equal(err.http_status, 503);
done(); done();

302
test/acceptance/multilayer.js
View File

@ -17,6 +17,8 @@ var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions); var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0); server.setMaxListeners(0);
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) { ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
var suiteName = 'multilayer:postgres=layergroup_url=' + layergroup_url; var suiteName = 'multilayer:postgres=layergroup_url=' + layergroup_url;
@ -67,17 +69,14 @@ suite(suiteName, function() {
assert.equal(res.statusCode, 200, res.body); assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body); var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.last_updated, expected_last_updated); assert.equal(parsedBody.last_updated, expected_last_updated);
if ( expected_token ) { assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); expected_token = parsedBody.layergroupid.split(':')[0];
assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
}
else expected_token = parsedBody.layergroupid.split(':')[0];
next(null, res); next(null, res);
}); });
}, },
function do_get_tile(err) function do_get_tile(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png', url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
@ -122,7 +121,7 @@ suite(suiteName, function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/170 // See https://github.com/CartoDB/Windshaft-cartodb/issues/170
function do_get_tile_nosignature(err) function do_get_tile_nosignature(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + '/localhost@' + expected_token + ':cb0/0/0/0.png', url: layergroup_url + '/localhost@' + expected_token + ':cb0/0/0/0.png',
@ -139,7 +138,7 @@ suite(suiteName, function() {
}, },
function do_get_grid_layer0(err) function do_get_grid_layer0(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json', url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json',
@ -156,7 +155,7 @@ suite(suiteName, function() {
}, },
function do_get_grid_layer1(err) function do_get_grid_layer1(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json', url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json',
@ -178,12 +177,20 @@ suite(suiteName, function() {
console.log("Error: " + err); console.log("Error: " + err);
} }
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
if ( errors.length ) done(new Error(errors)); errors.push(err.message);
else done(null); }
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
}); });
}); });
} }
@ -226,17 +233,23 @@ suite(suiteName, function() {
test("get creation requests has cache", function(done) { test("get creation requests has cache", function(done) {
var layergroup = { var layergroup = {
version: '1.0.0', version: '1.0.0',
layers: [ layers: [
{ options: { { options: {
sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' + sql: 'select cartodb_id, the_geom_webmercator from test_table',
' from test_table limit 2', cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', cartocss_version: '2.0.1',
cartocss_version: '2.0.1' interactivity: 'cartodb_id'
} } } },
] { options: {
}; sql: 'select cartodb_id, the_geom_webmercator from test_table_2',
cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
cartocss_version: '2.0.2',
interactivity: 'cartodb_id'
} }
]
};
var expected_token; var expected_token;
step( step(
@ -250,11 +263,15 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); }); }, {}, function(res, err) { next(err, res); });
}, },
function do_check_create(err, res) { function do_check_create(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 200, res.body); assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body); var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0]; expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res); helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
'public.test_table',
'public.test_table_2'
]).key().join(' '));
return null; return null;
}, },
function finish(err) { function finish(err) {
@ -264,12 +281,20 @@ suite(suiteName, function() {
console.log("Error: " + err); console.log("Error: " + err);
} }
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
if ( errors.length ) done(new Error(errors)); errors.push(err.message);
else done(null); }
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
}); });
}); });
} }
@ -353,13 +378,15 @@ suite(suiteName, function() {
if ( expected_token ) { if ( expected_token ) {
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
} }
else expected_token = parsedBody.layergroupid.split(':')[0]; else {
expected_token = parsedBody.layergroupid.split(':')[0];
}
next(null, res); next(null, res);
}); });
}, },
function do_get_tile1(err) function do_get_tile1(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb10/1/0/0.png', url: layergroup_url + "/" + expected_token + ':cb10/1/0/0.png',
@ -398,7 +425,7 @@ suite(suiteName, function() {
}, },
function do_get_tile4(err) function do_get_tile4(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb11/4/0/0.png', url: layergroup_url + "/" + expected_token + ':cb11/4/0/0.png',
@ -437,7 +464,7 @@ suite(suiteName, function() {
}, },
function do_get_grid1(err) function do_get_grid1(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/1/0/0.grid.json', url: layergroup_url + "/" + expected_token + '/0/1/0/0.grid.json',
@ -454,7 +481,7 @@ suite(suiteName, function() {
}, },
function do_get_grid4(err) function do_get_grid4(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/4/0/0.grid.json', url: layergroup_url + "/" + expected_token + '/0/4/0/0.grid.json',
@ -476,12 +503,20 @@ suite(suiteName, function() {
console.log("Error: " + err); console.log("Error: " + err);
} }
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
if ( errors.length ) done(new Error(errors)); errors.push(err.message);
else done(null); }
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
}); });
}); });
} }
@ -511,13 +546,17 @@ suite(suiteName, function() {
{ {
var next = this; var next = this;
redis_stats_client.select(redis_stats_db, function(err) { redis_stats_client.select(redis_stats_db, function(err) {
if ( err ) next(err); if ( err ) {
else redis_stats_client.del(statskey+':global', next); next(err);
}
else {
redis_stats_client.del(statskey+':global', next);
}
}); });
}, },
function do_post_1(err) function do_post_1(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url, url: layergroup_url,
@ -531,12 +570,12 @@ suite(suiteName, function() {
}); });
}, },
function check_global_stats_1(err, val) { function check_global_stats_1(err, val) {
if ( err ) throw err; assert.ifError(err);
assert.equal(val, 1, "Expected score of " + now + " in " + statskey + ":global to be 1, got " + val); assert.equal(val, 1, "Expected score of " + now + " in " + statskey + ":global to be 1, got " + val);
redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this); redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this);
}, },
function check_tag_stats_1_do_post_2(err, val) { function check_tag_stats_1_do_post_2(err, val) {
if ( err ) throw err; assert.ifError(err);
assert.equal(val, 1, "Expected score of " + now + " in " + statskey + ":stat_tag:" + layergroup.stat_tag + assert.equal(val, 1, "Expected score of " + now + " in " + statskey + ":stat_tag:" + layergroup.stat_tag +
" to be 1, got " + val); " to be 1, got " + val);
var next = this; var next = this;
@ -553,19 +592,21 @@ suite(suiteName, function() {
}, },
function check_global_stats_2(err, val) function check_global_stats_2(err, val)
{ {
if ( err ) throw err; assert.ifError(err);
assert.equal(val, 2, "Expected score of " + now + " in " + statskey + ":global to be 2, got " + val); assert.equal(val, 2, "Expected score of " + now + " in " + statskey + ":global to be 2, got " + val);
redis_stats_client.zscore(statskey+':stat_tag:' + layergroup.stat_tag, now, this); redis_stats_client.zscore(statskey+':stat_tag:' + layergroup.stat_tag, now, this);
}, },
function check_tag_stats_2(err, val) function check_tag_stats_2(err, val)
{ {
if ( err ) throw err; assert.ifError(err);
assert.equal(val, 2, "Expected score of " + now + " in " + statskey + ":stat_tag:" + layergroup.stat_tag + assert.equal(val, 2, "Expected score of " + now + " in " + statskey + ":stat_tag:" + layergroup.stat_tag +
" to be 2, got " + val); " to be 2, got " + val);
return 1; return 1;
}, },
function cleanup_map_style(err) { function cleanup_map_style(err) {
if ( err ) errors.push('' + err); if ( err ) {
errors.push('' + err);
}
var next = this; var next = this;
// trip epoch // trip epoch
expected_token = expected_token.split(':')[0]; expected_token = expected_token.split(':')[0];
@ -574,13 +615,21 @@ suite(suiteName, function() {
}); });
}, },
function cleanup_stats(err) { function cleanup_stats(err) {
if ( err ) errors.push('' + err); if ( err ) {
errors.push('' + err);
}
redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this); redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this);
}, },
function finish(err) { function finish(err) {
if ( err ) errors.push('' + err); if ( err ) {
if ( errors.length ) done(new Error(errors.join(','))); errors.push('' + err);
else done(null); }
if ( errors.length ) {
done(new Error(errors.join(',')));
}
else {
done(null);
}
} }
); );
}); });
@ -678,13 +727,15 @@ suite(suiteName, function() {
if ( expected_token ) { if ( expected_token ) {
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
} }
else expected_token = parsedBody.layergroupid.split(':')[0]; else {
expected_token = parsedBody.layergroupid.split(':')[0];
}
next(null, res); next(null, res);
}); });
}, },
function do_get_tile(err) function do_get_tile(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234', url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234',
@ -705,7 +756,7 @@ suite(suiteName, function() {
}, },
function do_get_grid_layer0(err) function do_get_grid_layer0(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json?map_key=1234', url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json?map_key=1234',
@ -718,7 +769,7 @@ suite(suiteName, function() {
}, },
function do_get_grid_layer1(err) function do_get_grid_layer1(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json?map_key=1234', url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json?map_key=1234',
@ -732,7 +783,7 @@ suite(suiteName, function() {
}, },
function do_get_tile_unauth(err) function do_get_tile_unauth(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png', url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
@ -748,7 +799,7 @@ suite(suiteName, function() {
}, },
function do_get_grid_layer0_unauth(err) function do_get_grid_layer0_unauth(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json', url: layergroup_url + "/" + expected_token + '/0/0/0/0.grid.json',
@ -763,7 +814,7 @@ suite(suiteName, function() {
}, },
function do_get_grid_layer1_unauth(err) function do_get_grid_layer1_unauth(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json', url: layergroup_url + "/" + expected_token + '/1/0/0/0.grid.json',
@ -783,12 +834,20 @@ suite(suiteName, function() {
console.log("Error: " + err); console.log("Error: " + err);
} }
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
if ( errors.length ) done(new Error(errors)); errors.push(err.message);
else done(null); }
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
}); });
}); });
} }
@ -823,19 +882,21 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); }); }, {}, function(res, err) { next(err, res); });
}, },
function check_post(err, res) { function check_post(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 200, res.body); assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body); var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.last_updated, expected_last_updated); assert.equal(parsedBody.last_updated, expected_last_updated);
if ( expected_token ) { if ( expected_token ) {
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
} }
else expected_token = parsedBody.layergroupid.split(':')[0]; else {
expected_token = parsedBody.layergroupid.split(':')[0];
}
return null; return null;
}, },
function do_get0(err) function do_get0(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234', url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234',
@ -845,7 +906,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); }); }, {}, function(res, err) { next(err, res); });
}, },
function do_check0(err, res) { function do_check0(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 200, res.body); assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png"); assert.equal(res.headers['content-type'], "image/png");
@ -857,14 +918,14 @@ suite(suiteName, function() {
return null; return null;
}, },
function do_restart_server(err/*, res*/) { function do_restart_server(err/*, res*/) {
if ( err ) throw err; assert.ifError(err);
// hack simulating restart... // hack simulating restart...
server = new CartodbWindshaft(serverOptions); server = new CartodbWindshaft(serverOptions);
return null; return null;
}, },
function do_get1(err) function do_get1(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234', url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?map_key=1234',
@ -874,7 +935,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); }); }, {}, function(res, err) { next(err, res); });
}, },
function do_check1(err, res) { function do_check1(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 200, res.body); assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png"); assert.equal(res.headers['content-type'], "image/png");
@ -892,12 +953,20 @@ suite(suiteName, function() {
console.log("Error: " + err); console.log("Error: " + err);
} }
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
if ( errors.length ) done(new Error(errors.join(','))); errors.push(err.message);
else done(null); }
if ( errors.length ) {
done(new Error(errors.join(',')));
}
else {
done(null);
}
}); });
}); });
} }
@ -1024,7 +1093,7 @@ suite(suiteName, function() {
}, },
function do_get_tile(err) function do_get_tile(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png', url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
@ -1048,12 +1117,20 @@ suite(suiteName, function() {
console.log("Error: " + err); console.log("Error: " + err);
} }
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
if ( errors.length ) done(new Error(errors)); errors.push(err.message);
else done(null); }
if ( errors.length ) {
done(new Error(errors));
}
else {
done(null);
}
}); });
}); });
} }
@ -1087,7 +1164,7 @@ suite(suiteName, function() {
}, {}, function(res) { next(null, res); }); }, {}, function(res) { next(null, res); });
}, },
function check_result(err, res) { function check_result(err, res) {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsedBody = JSON.parse(res.body); var parsedBody = JSON.parse(res.body);
@ -1104,7 +1181,7 @@ suite(suiteName, function() {
}, },
function do_get_tile(err) function do_get_tile(err)
{ {
if ( err ) throw err; assert.ifError(err);
var next = this; var next = this;
assert.response(server, { assert.response(server, {
url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?api_key=1234', url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png?api_key=1234',
@ -1114,19 +1191,27 @@ suite(suiteName, function() {
}, {}, function(res) { next(null, res); }); }, {}, function(res) { next(null, res); });
}, },
function check_get_tile(err, res) { function check_get_tile(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 200, res.body); assert.equal(res.statusCode, 200, res.body);
return null; return null;
}, },
function cleanup(err) { function cleanup(err) {
if ( err ) errors.push(err.message); if ( err ) {
if ( ! expected_token ) return null; errors.push(err.message);
}
if ( ! expected_token ) {
return null;
}
var next = this; var next = this;
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
next(); next();
}); });
}); });
@ -1136,8 +1221,12 @@ suite(suiteName, function() {
errors.push(err.message); errors.push(err.message);
console.log("Error: " + err); console.log("Error: " + err);
} }
if ( errors.length ) done(new Error(errors)); if ( errors.length ) {
else done(null); done(new Error(errors));
}
else {
done(null);
}
} }
); );
}); });
@ -1146,11 +1235,14 @@ suite(suiteName, function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/111 // See https://github.com/CartoDB/Windshaft-cartodb/issues/111
test("sql string can be very long", function(done){ test("sql string can be very long", function(done){
var long_val = 'pretty'; var long_val = 'pretty';
for (var i=0; i<1024; ++i) long_val += ' long'; for (var i=0; i<1024; ++i) {
long_val += ' long';
}
long_val += ' string'; long_val += ' string';
var sql = "SELECT "; var sql = "SELECT ";
for (i=0; i<16; ++i) for (i=0; i<16; ++i) {
sql += "'" + long_val + "'::text as pretty_long_field_name_" + i + ", "; sql += "'" + long_val + "'::text as pretty_long_field_name_" + i + ", ";
}
sql += "cartodb_id, the_geom_webmercator FROM gadm4 g"; sql += "cartodb_id, the_geom_webmercator FROM gadm4 g";
var layergroup = { var layergroup = {
version: '1.0.0', version: '1.0.0',
@ -1178,7 +1270,7 @@ suite(suiteName, function() {
}, {}, function(res) { next(null, res); }); }, {}, function(res) { next(null, res); });
}, },
function check_result(err, res) { function check_result(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsedBody = JSON.parse(res.body); var parsedBody = JSON.parse(res.body);
var token_components = parsedBody.layergroupid.split(':'); var token_components = parsedBody.layergroupid.split(':');
@ -1186,22 +1278,36 @@ suite(suiteName, function() {
return null; return null;
}, },
function cleanup(err) { function cleanup(err) {
if ( err ) errors.push('' + err); if ( err ) {
if ( ! expected_token ) return null; errors.push('' + err);
}
if ( ! expected_token ) {
return null;
}
var next = this; var next = this;
redis_client.keys("map_cfg|" + expected_token, function(err, matches) { redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) { redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message); if ( err ) {
errors.push(err.message);
}
next(); next();
}); });
}); });
}, },
function finish(err) { function finish(err) {
if ( err ) errors.push('' + err); if ( err ) {
if ( errors.length ) done(new Error(errors.join(','))); errors.push('' + err);
else done(null); }
if ( errors.length ) {
done(new Error(errors.join(',')));
}
else {
done(null);
}
} }
); );
}); });
@ -1232,7 +1338,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); }); }, {}, function(res, err) { next(err, res); });
}, },
function check_post(err, res) { function check_post(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body); var parsed = JSON.parse(res.body);
assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed)); assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed));
@ -1274,7 +1380,7 @@ suite(suiteName, function() {
}, {}, function(res, err) { next(err, res); }); }, {}, function(res, err) { next(err, res); });
}, },
function check_post(err, res) { function check_post(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body); var parsed = JSON.parse(res.body);
assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed)); assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed));

4
test/acceptance/multilayer_server.js
View File

@ -318,7 +318,7 @@ describe('tests from old api translated to multilayer', function() {
var parsed = JSON.parse(res.body); var parsed = JSON.parse(res.body);
assert.deepEqual(parsed, { assert.deepEqual(parsed, {
errors: ["Error: could not fetch affected tables and last updated time: fake error message"] errors: ["Error: could not fetch affected tables or last updated time: fake error message"]
}); });
done(); done();
@ -346,7 +346,7 @@ describe('tests from old api translated to multilayer', function() {
}; };
// reset internal cacheChannel cache // reset internal cacheChannel cache
server.channelCache = {}; server.layergroupAffectedTablesCache.cache.reset();
assert.response(server, assert.response(server,
{ {

4
test/acceptance/ported/support/ported_server_options.js
View File

@ -58,6 +58,10 @@ module.exports = _.extend({}, serverOptions, {
_.extend(req.params, req.query); _.extend(req.params, req.query);
req.params.user = 'localhost'; req.params.user = 'localhost';
req.context = {user: 'localhost'}; req.context = {user: 'localhost'};
req.params.dbhost = global.environment.postgres.host;
req.params.dbport = req.params.dbport || global.environment.postgres.port;
req.params.dbuser = 'test_windshaft_publicuser'; req.params.dbuser = 'test_windshaft_publicuser';
if (req.params.dbname !== 'windshaft_test2') { if (req.params.dbname !== 'windshaft_test2') {
req.params.dbuser = 'test_windshaft_cartodb_user_1'; req.params.dbuser = 'test_windshaft_cartodb_user_1';

2
test/acceptance/server.js
View File

@ -23,7 +23,7 @@ suite('server', function() {
},{}, function(res, err) { next(err,res); }); },{}, function(res, err) { next(err,res); });
}, },
function doCheck(err, res) { function doCheck(err, res) {
if ( err ) throw err; assert.ifError(err);
assert.ok(res.statusCode, 200); assert.ok(res.statusCode, 200);
var cc = res.headers['x-cache-channel']; var cc = res.headers['x-cache-channel'];
assert.ok(!cc); assert.ok(!cc);

453
test/acceptance/templates.js
View File

File diff suppressed because it is too large Load Diff

4
test/support/sql/CDB_QueryTables.sql
View File

@ -41,11 +41,11 @@ BEGIN
xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x, xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
) )
SELECT unnest(x) as p, unnest(s) as sc from inp SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
LOOP LOOP
-- RAISE DEBUG 'tab: %', rec2.p; -- RAISE DEBUG 'tab: %', rec2.p;
-- RAISE DEBUG 'sc: %', rec2.sc; -- RAISE DEBUG 'sc: %', rec2.sc;
tables := array_append(tables, (rec2.sc || '.' || rec2.p)); tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
END LOOP; END LOOP;
-- RAISE DEBUG 'Tables: %', tables; -- RAISE DEBUG 'Tables: %', tables;

1
test/support/test_helper.js
View File

@ -34,6 +34,7 @@ function lzma_compress_to_base64(payload, mode, callback) {
// Throws on failure // Throws on failure
function checkNoCache(res) { function checkNoCache(res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
assert.ok(!res.headers.hasOwnProperty('surrogate-key'));
assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ? assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ?
assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ? assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ?
} }

77
test/unit/cartodb/template_maps.test.js
View File

@ -202,7 +202,7 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this); tmap.addTemplate('me', tpl, this);
}, },
function addOmonimousTemplate(err, id) { function addOmonimousTemplate(err, id) {
if ( err ) throw err; assert.ifError(err);
tpl_id = id; tpl_id = id;
assert.equal(tpl_id, 'first'); assert.equal(tpl_id, 'first');
expected_failure = true; expected_failure = true;
@ -210,13 +210,15 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this); tmap.addTemplate('me', tpl, this);
}, },
function getTemplate(err) { function getTemplate(err) {
if ( ! expected_failure && err ) throw err; if ( ! expected_failure && err ) {
throw err;
}
assert.ok(err); assert.ok(err);
assert.ok(err.message.match(/already exists/i), err); assert.ok(err.message.match(/already exists/i), err);
tmap.getTemplate('me', tpl_id, this); tmap.getTemplate('me', tpl_id, this);
}, },
function delTemplate(err, got_tpl) { function delTemplate(err, got_tpl) {
if ( err ) throw err; assert.ifError(err);
assert.deepEqual(got_tpl, _.extend({}, tpl, {auth: {method: 'open'}, placeholders: {}})); assert.deepEqual(got_tpl, _.extend({}, tpl, {auth: {method: 'open'}, placeholders: {}}));
tmap.delTemplate('me', tpl_id, this); tmap.delTemplate('me', tpl_id, this);
}, },
@ -238,31 +240,35 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl1, this); tmap.addTemplate('me', tpl1, this);
}, },
function addTemplate2(err, id) { function addTemplate2(err, id) {
if ( err ) throw err; assert.ifError(err);
tpl1_id = id; tpl1_id = id;
tmap.addTemplate('me', tpl2, this); tmap.addTemplate('me', tpl2, this);
}, },
function listTemplates(err, id) { function listTemplates(err, id) {
if ( err ) throw err; assert.ifError(err);
tpl2_id = id; tpl2_id = id;
tmap.listTemplates('me', this); tmap.listTemplates('me', this);
}, },
function checkTemplates(err, ids) { function checkTemplates(err, ids) {
if ( err ) throw err; assert.ifError(err);
assert.equal(ids.length, 2); assert.equal(ids.length, 2);
assert.ok(ids.indexOf(tpl1_id) != -1, ids.join(',')); assert.ok(ids.indexOf(tpl1_id) !== -1, ids.join(','));
assert.ok(ids.indexOf(tpl2_id) != -1, ids.join(',')); assert.ok(ids.indexOf(tpl2_id) !== -1, ids.join(','));
return null; return null;
}, },
function delTemplate1(err) { function delTemplate1(err) {
if ( tpl1_id ) { if ( tpl1_id ) {
var next = this; var next = this;
tmap.delTemplate('me', tpl1_id, function(e) { tmap.delTemplate('me', tpl1_id, function(e) {
if ( err || e ) next(new Error(err + '; ' + e)); if ( err || e ) {
else next(); next(new Error(err + '; ' + e));
}
else {
next();
}
}); });
} else { } else {
if ( err ) throw err; assert.ifError(err);
return null; return null;
} }
}, },
@ -270,11 +276,15 @@ describe('template_maps', function() {
if ( tpl2_id ) { if ( tpl2_id ) {
var next = this; var next = this;
tmap.delTemplate('me', tpl2_id, function(e) { tmap.delTemplate('me', tpl2_id, function(e) {
if ( err || e ) next(new Error(err + '; ' + e)); if ( err || e ) {
else next(); next(new Error(err + '; ' + e));
}
else {
next();
}
}); });
} else { } else {
if ( err ) throw err; assert.ifError(err);
return null; return null;
} }
}, },
@ -301,14 +311,16 @@ describe('template_maps', function() {
}, },
// Updating template name should fail // Updating template name should fail
function updateTemplateName(err, id) { function updateTemplateName(err, id) {
if ( err ) throw err; assert.ifError(err);
tpl_id = id; tpl_id = id;
expected_failure = true; expected_failure = true;
tpl.name = 'second'; tpl.name = 'second';
tmap.updTemplate(owner, tpl_id, tpl, this); tmap.updTemplate(owner, tpl_id, tpl, this);
}, },
function updateTemplateAuth(err) { function updateTemplateAuth(err) {
if ( err && ! expected_failure) throw err; if ( err && ! expected_failure) {
throw err;
}
expected_failure = false; expected_failure = false;
assert.ok(err); assert.ok(err);
tpl.name = 'first'; tpl.name = 'first';
@ -317,13 +329,15 @@ describe('template_maps', function() {
tmap.updTemplate(owner, tpl_id, tpl, this); tmap.updTemplate(owner, tpl_id, tpl, this);
}, },
function updateTemplateWithInvalid(err) { function updateTemplateWithInvalid(err) {
if ( err ) throw err; assert.ifError(err);
tpl.version = '999.999.999'; tpl.version = '999.999.999';
expected_failure = true; expected_failure = true;
tmap.updTemplate(owner, tpl_id, tpl, this); tmap.updTemplate(owner, tpl_id, tpl, this);
}, },
function updateUnexistentTemplate(err) { function updateUnexistentTemplate(err) {
if ( err && ! expected_failure) throw err; if ( err && ! expected_failure) {
throw err;
}
assert.ok(err); assert.ok(err);
assert.ok(err.message.match(/unsupported.*version/i), err); assert.ok(err.message.match(/unsupported.*version/i), err);
tpl.version = '0.0.1'; tpl.version = '0.0.1';
@ -331,7 +345,9 @@ describe('template_maps', function() {
tmap.updTemplate(owner, 'unexistent', tpl, this); tmap.updTemplate(owner, 'unexistent', tpl, this);
}, },
function delTemplate(err) { function delTemplate(err) {
if ( err && ! expected_failure) throw err; if ( err && ! expected_failure) {
throw err;
}
expected_failure = false; expected_failure = false;
assert.ok(err); assert.ok(err);
assert.ok(err.message.match(/cannot update name/i), err); assert.ok(err.message.match(/cannot update name/i), err);
@ -344,6 +360,7 @@ describe('template_maps', function() {
}); });
it('instanciate templates', function() { it('instanciate templates', function() {
// jshint maxcomplexity:7
var tmap = new TemplateMaps(redis_pool); var tmap = new TemplateMaps(redis_pool);
assert.ok(tmap); assert.ok(tmap);
@ -456,14 +473,14 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this); tmap.addTemplate('me', tpl, this);
}, },
function twoForMe(err, id) { function twoForMe(err, id) {
if ( err ) throw err; assert.ifError(err);
assert.ok(id); assert.ok(id);
idMe.push(id); idMe.push(id);
tpl.name = 'twoForMe'; tpl.name = 'twoForMe';
tmap.addTemplate('me', tpl, this); tmap.addTemplate('me', tpl, this);
}, },
function threeForMe(err, id) { function threeForMe(err, id) {
if ( err ) throw err; assert.ifError(err);
assert.ok(id); assert.ok(id);
idMe.push(id); idMe.push(id);
tpl.name = 'threeForMe'; tpl.name = 'threeForMe';
@ -471,37 +488,39 @@ describe('template_maps', function() {
tmap.addTemplate('me', tpl, this); tmap.addTemplate('me', tpl, this);
}, },
function errForMe(err/*, id*/) { function errForMe(err/*, id*/) {
if ( err && ! expectErr ) throw err; if ( err && ! expectErr ) {
throw err;
}
expectErr = false; expectErr = false;
assert.ok(err); assert.ok(err);
assert.ok(err.message.match(/limit.*template/), err); assert.ok(err.message.match(/limit.*template/), err);
return null; return null;
}, },
function delOneMe(err) { function delOneMe(err) {
if ( err ) throw err; assert.ifError(err);
tmap.delTemplate('me', idMe.shift(), this); tmap.delTemplate('me', idMe.shift(), this);
}, },
function threeForMeRetry(err) { function threeForMeRetry(err) {
if ( err ) throw err; assert.ifError(err);
tpl.name = 'threeForMe'; tpl.name = 'threeForMe';
tmap.addTemplate('me', tpl, this); tmap.addTemplate('me', tpl, this);
}, },
function oneForYou(err, id) { function oneForYou(err, id) {
if ( err ) throw err; assert.ifError(err);
assert.ok(id); assert.ok(id);
idMe.push(id); idMe.push(id);
tpl.name = 'oneForYou'; tpl.name = 'oneForYou';
tmap.addTemplate('you', tpl, this); tmap.addTemplate('you', tpl, this);
}, },
function twoForYou(err, id) { function twoForYou(err, id) {
if ( err ) throw err; assert.ifError(err);
assert.ok(id); assert.ok(id);
idYou.push(id); idYou.push(id);
tpl.name = 'twoForYou'; tpl.name = 'twoForYou';
tmap.addTemplate('you', tpl, this); tmap.addTemplate('you', tpl, this);
}, },
function threeForYou(err, id) { function threeForYou(err, id) {
if ( err ) throw err; assert.ifError(err);
assert.ok(id); assert.ok(id);
idYou.push(id); idYou.push(id);
tpl.name = 'threeForYou'; tpl.name = 'threeForYou';
@ -509,7 +528,9 @@ describe('template_maps', function() {
tmap.addTemplate('you', tpl, this); tmap.addTemplate('you', tpl, this);
}, },
function errForYou(err/*, id*/) { function errForYou(err/*, id*/) {
if ( err && ! expectErr ) throw err; if ( err && ! expectErr ) {
throw err;
}
expectErr = false; expectErr = false;
assert.ok(err); assert.ok(err);
assert.ok(err.message.match(/limit.*template/), err); assert.ok(err.message.match(/limit.*template/), err);