cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

QueryTables and last updated_at retrieved with user

Browse Source 
Move setDBAuth and setDBConn to PgConnection entity
 - It uses cartodb-redis to retrieve datasource configuration
Start using it in ServerOptions, TemplateMaps and QueryTablesApi
QueryTablesApi don't receive anymore the connection/credentials
 - It will always use an authenticated query to retrieve last update
 - That will allow to query affected private tables last update
This commit is contained in:
Raul Ochoa 2015-02-09 14:46:52 +01:00
parent 04af57cab9
commit 90b22b2718
5 changed files with 183 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
lib/cartodb/api/query_tables_api.js
View File

@ -1,7 +1,10 @@
var sqlApi = require('../sql/sql_api'),
PSQL = require('cartodb-psql');
var sqlApi = require('../sql/sql_api');
var PSQL = require('cartodb-psql');
var Step = require('step');
function QueryTablesApi() {
function QueryTablesApi(pgConnection, metadataBackend) {
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
}
var affectedTableRegexCache = {
@ -14,11 +17,11 @@ var affectedTableRegexCache = {
module.exports = QueryTablesApi;
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
runQuery(username, options, query, handleAffectedTablesInQueryRows, callback);
this.runQuery(username, query, handleAffectedTablesInQueryRows, callback);
};
function handleAffectedTablesInQueryRows(err, rows, callback) {
@ -33,7 +36,7 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
callback(null, tableNames);
}
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
var query = [
'WITH querytables AS (',
@ -44,7 +47,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
this.runQuery(username, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
};
function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
@ -68,20 +71,60 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
}
function runQuery(username, options, query, queryHandler, callback) {
QueryTablesApi.prototype.runQuery = function(username, query, queryHandler, callback) {
var self = this;
if (shouldQueryPostgresDirectly()) {
var psql = new PSQL(options);
psql.query(query, function(err, resultSet) {
resultSet = resultSet || {};
var rows = resultSet.rows || [];
queryHandler(err, rows, callback);
});
var params = {};
Step(
function setAuth() {
self.pgConnection.setDBAuth(username, params, this);
},
function setConn(err) {
if (err) {
throw err;
}
self.pgConnection.setDBConn(username, params, this);
},
function executeQuery(err) {
if (err) {
throw err;
}
var psql = new PSQL({
user: params.dbuser,
pass: params.dbpass,
host: params.dbhost,
port: params.dbport,
dbname: params.dbname
});
psql.query(query, function(err, resultSet) {
resultSet = resultSet || {};
var rows = resultSet.rows || [];
queryHandler(err, rows, callback);
});
}
);
} else {
sqlApi.query(username, options.api_key, query, function(err, rows) {
queryHandler(err, rows, callback);
});
Step(
function getApiKey() {
self.metadataBackend.getUserMapKey(username, this);
},
function executeQuery(err, apiKey) {
if (err) {
throw err;
}
sqlApi.query(username, apiKey, query, function(err, rows) {
queryHandler(err, rows, callback);
});
}
);
}
}
};
function prepareSql(sql) {

96
lib/cartodb/backends/pg_connection.js Normal file
View File

@ -0,0 +1,96 @@
var Step = require('step');
var _ = require('underscore');
function PgConnection(metadataBackend) {
this.metadataBackend = metadataBackend;
}
module.exports = PgConnection;
// Set db authentication parameters to those of the given username
//
// @param username the cartodb username, mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set auth options into
// added params are: "dbuser" and "dbpassword"
//
// @param callback function(err)
//
PgConnection.prototype.setDBAuth = function(username, params, callback) {
var self = this;
var user_params = {};
var auth_user = global.environment.postgres_auth_user;
var auth_pass = global.environment.postgres_auth_pass;
Step(
function getId() {
self.metadataBackend.getUserId(username, this);
},
function(err, user_id) {
if (err) throw err;
user_params['user_id'] = user_id;
var dbuser = _.template(auth_user, user_params);
_.extend(params, {dbuser:dbuser});
// skip looking up user_password if postgres_auth_pass
// doesn't contain the "user_password" label
if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
self.metadataBackend.getUserDBPass(username, this);
},
function(err, user_password) {
if (err) throw err;
user_params['user_password'] = user_password;
if ( auth_pass ) {
var dbpass = _.template(auth_pass, user_params);
_.extend(params, {dbpassword:dbpass});
}
return true;
},
function finish(err) {
callback(err);
}
);
};
// Set db connection parameters to those for the given username
//
// @param dbowner cartodb username of database owner,
// mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set connection options into
// added params are: "dbname", "dbhost"
//
// @param callback function(err)
//
PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
var self = this;
// Add default database connection parameters
// if none given
_.defaults(params, {
dbuser: global.environment.postgres.user,
dbpassword: global.environment.postgres.password,
dbhost: global.environment.postgres.host,
dbport: global.environment.postgres.port
});
Step(
function getConnectionParams() {
self.metadataBackend.getUserDBConnectionParams(dbowner, this);
},
function extendParams(err, dbParams){
if (err) throw err;
// we don't want null values or overwrite a non public user
if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
delete dbParams.dbuser;
}
if ( dbParams ) _.extend(params, dbParams);
return null;
},
function finish(err) {
callback(err);
}
);
};

9
lib/cartodb/cartodb_windshaft.js
View File

@ -158,7 +158,14 @@ var CartodbWindshaft = function(serverOptions) {
var TemplateMapsController = require('./controllers/template_maps'),
templateMapsController = new TemplateMapsController(
ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry
ws,
serverOptions,
templateMaps,
cartoData,
template_baseurl,
surrogateKeysCache,
NamedMapsCacheEntry,
serverOptions.pgConnection
);
templateMapsController.register(ws);

7
lib/cartodb/controllers/template_maps.js
View File

@ -2,7 +2,7 @@ var Step = require('step');
var _ = require('underscore');
function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache,
NamedMapsCacheEntry) {
NamedMapsCacheEntry, pgConnection) {
this.app = app;
this.serverOptions = serverOptions;
this.templateMaps = templateMaps;
@ -10,6 +10,7 @@ function TemplateMapsController(app, serverOptions, templateMaps, metadataBacken
this.templateBaseUrl = templateBaseUrl;
this.surrogateKeysCache = surrogateKeysCache;
this.NamedMapsCacheEntry = NamedMapsCacheEntry;
this.pgConnection = pgConnection;
}
module.exports = TemplateMapsController;
@ -465,11 +466,11 @@ TemplateMapsController.prototype.setDBParams = function(cdbuser, params, callbac
var self = this;
Step(
function setAuth() {
self.serverOptions.setDBAuth(cdbuser, params, this);
self.pgConnection.setDBAuth(cdbuser, params, this);
},
function setConn(err) {
if ( err ) throw err;
self.serverOptions.setDBConn(cdbuser, params, this);
self.pgConnection.setDBConn(cdbuser, params, this);
},
function finish(err) {
callback(err);

128
lib/cartodb/server_options.js
View File

@ -1,6 +1,7 @@
var _ = require('underscore');
var Step = require('step');
var QueryTablesApi = require('./api/query_tables_api');
var PgConnection = require('./backends/pg_connection');
var crypto = require('crypto');
var LZMA = require('lzma').LZMA;
@ -35,7 +36,8 @@ module.exports = function(redisPool) {
var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis;
var cartoData = require('cartodb-redis')(redisOpts),
lzmaWorker = new LZMA(),
queryTablesApi = new QueryTablesApi();
pgConnection = new PgConnection(cartoData),
queryTablesApi = new QueryTablesApi(pgConnection, cartoData);
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
@ -102,6 +104,9 @@ module.exports = function(redisPool) {
// Re-use redisPool
me.redis.pool = redisPool;
// Re-use pgConnection
me.pgConnection = pgConnection;
/* This whole block is about generating X-Cache-Channel { */
// TODO: review lifetime of elements of this cache
@ -211,14 +216,7 @@ module.exports = function(redisPool) {
if ( req.profiler ) req.profiler.done('getSignerMapKey');
key = data;
}
queryTablesApi.getAffectedTablesInQuery(user, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this); // in addCacheChannel
queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
},
function finish(err, data) {
next(err,data);
@ -329,28 +327,16 @@ module.exports = function(redisPool) {
done();
});
var sql = [];
_.each(mapconfig.layers, function(lyr) {
sql.push(lyr.options.sql);
});
sql = sql.join(';');
var sql = mapconfig.layers.map(function(layer) {
return layer.options.sql;
}).join(';');
var dbName = req.params.dbname;
var usr = this.userByReq(req);
var key = req.params.map_key || req.params.api_key;
var cacheKey = dbName + ':' + token;
Step(
function getAffectedTablesAndLastUpdatedTime() {
queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this);
queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
@ -405,92 +391,6 @@ module.exports = function(redisPool) {
return mat[1];
};
// Set db authentication parameters to those of the given username
//
// @param username the cartodb username, mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set auth options into
// added params are: "dbuser" and "dbpassword"
//
// @param callback function(err)
//
me.setDBAuth = function(username, params, callback) {
var user_params = {};
var auth_user = global.environment.postgres_auth_user;
var auth_pass = global.environment.postgres_auth_pass;
Step(
function getId() {
cartoData.getUserId(username, this);
},
function(err, user_id) {
if (err) throw err;
user_params['user_id'] = user_id;
var dbuser = _.template(auth_user, user_params);
_.extend(params, {dbuser:dbuser});
// skip looking up user_password if postgres_auth_pass
// doesn't contain the "user_password" label
if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
cartoData.getUserDBPass(username, this);
},
function(err, user_password) {
if (err) throw err;
user_params['user_password'] = user_password;
if ( auth_pass ) {
var dbpass = _.template(auth_pass, user_params);
_.extend(params, {dbpassword:dbpass});
}
return true;
},
function finish(err) {
callback(err);
}
);
};
// Set db connection parameters to those for the given username
//
// @param dbowner cartodb username of database owner,
// mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set connection options into
// added params are: "dbname", "dbhost"
//
// @param callback function(err)
//
me.setDBConn = function(dbowner, params, callback) {
// Add default database connection parameters
// if none given
_.defaults(params, {
dbuser: global.environment.postgres.user,
dbpassword: global.environment.postgres.password,
dbhost: global.environment.postgres.host,
dbport: global.environment.postgres.port
});
Step(
function getConnectionParams() {
cartoData.getUserDBConnectionParams(dbowner, this);
},
function extendParams(err, dbParams){
if (err) throw err;
// we don't want null values or overwrite a non public user
if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
delete dbParams.dbuser;
}
if ( dbParams ) _.extend(params, dbParams);
return null;
},
function finish(err) {
callback(err);
}
);
};
// Check if a request is authorized by a signer
//
// @param req express request object
@ -589,7 +489,7 @@ module.exports = function(redisPool) {
_.extend(req.params, { _authorizedByApiKey: true });
// authorized by api key, login as the given username and stop
that.setDBAuth(user, req.params, function(err) {
pgConnection.setDBAuth(user, req.params, function(err) {
callback(err, true); // authorized (or error)
});
},
@ -624,7 +524,7 @@ module.exports = function(redisPool) {
// Authorized by "signed_by" !
_.extend(req.params, { _authorizedBySigner: signed_by });
that.setDBAuth(signed_by, req.params, function(err) {
pgConnection.setDBAuth(signed_by, req.params, function(err) {
if (req.profiler) req.profiler.done('setDBAuth');
callback(err, true); // authorized (or error)
});
@ -738,7 +638,7 @@ module.exports = function(redisPool) {
},
function getDatabase(err){
if(err) throw err;
that.setDBConn(user, req.params, this);
pgConnection.setDBConn(user, req.params, this);
},
function getGeometryType(err){
if (req.profiler) req.profiler.done('setDBConn');