From 90b22b271824a834f472e94ced5177b40256dd69 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 9 Feb 2015 14:46:52 +0100 Subject: [PATCH] QueryTables and last updated_at retrieved with user Move setDBAuth and setDBConn to PgConnection entity - It uses cartodb-redis to retrieve datasource configuration Start using it in ServerOptions, TemplateMaps and QueryTablesApi QueryTablesApi don't receive anymore the connection/credentials - It will always use an authenticated query to retrieve last update - That will allow to query affected private tables last update --- lib/cartodb/api/query_tables_api.js | 79 ++++++++++---- lib/cartodb/backends/pg_connection.js | 96 +++++++++++++++++ lib/cartodb/cartodb_windshaft.js | 9 +- lib/cartodb/controllers/template_maps.js | 7 +- lib/cartodb/server_options.js | 128 +++-------------------- 5 files changed, 183 insertions(+), 136 deletions(-) create mode 100644 lib/cartodb/backends/pg_connection.js diff --git a/lib/cartodb/api/query_tables_api.js b/lib/cartodb/api/query_tables_api.js index bdcf488e..f4795b05 100644 --- a/lib/cartodb/api/query_tables_api.js +++ b/lib/cartodb/api/query_tables_api.js @@ -1,7 +1,10 @@ -var sqlApi = require('../sql/sql_api'), - PSQL = require('cartodb-psql'); +var sqlApi = require('../sql/sql_api'); +var PSQL = require('cartodb-psql'); +var Step = require('step'); -function QueryTablesApi() { +function QueryTablesApi(pgConnection, metadataBackend) { + this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; } var affectedTableRegexCache = { @@ -14,11 +17,11 @@ var affectedTableRegexCache = { module.exports = QueryTablesApi; -QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) { +QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) { var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)'; - runQuery(username, options, query, handleAffectedTablesInQueryRows, callback); + this.runQuery(username, query, handleAffectedTablesInQueryRows, callback); }; function handleAffectedTablesInQueryRows(err, rows, callback) { @@ -33,7 +36,7 @@ function handleAffectedTablesInQueryRows(err, rows, callback) { callback(null, tableNames); } -QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) { +QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) { var query = [ 'WITH querytables AS (', @@ -44,7 +47,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam 'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])' ].join(' '); - runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback); + this.runQuery(username, query, handleAffectedTablesAndLastUpdatedTimeRows, callback); }; function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { @@ -68,20 +71,60 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { } -function runQuery(username, options, query, queryHandler, callback) { +QueryTablesApi.prototype.runQuery = function(username, query, queryHandler, callback) { + var self = this; + if (shouldQueryPostgresDirectly()) { - var psql = new PSQL(options); - psql.query(query, function(err, resultSet) { - resultSet = resultSet || {}; - var rows = resultSet.rows || []; - queryHandler(err, rows, callback); - }); + + var params = {}; + + Step( + function setAuth() { + self.pgConnection.setDBAuth(username, params, this); + }, + function setConn(err) { + if (err) { + throw err; + } + self.pgConnection.setDBConn(username, params, this); + }, + function executeQuery(err) { + if (err) { + throw err; + } + var psql = new PSQL({ + user: params.dbuser, + pass: params.dbpass, + host: params.dbhost, + port: params.dbport, + dbname: params.dbname + }); + psql.query(query, function(err, resultSet) { + resultSet = resultSet || {}; + var rows = resultSet.rows || []; + queryHandler(err, rows, callback); + }); + } + ); + } else { - sqlApi.query(username, options.api_key, query, function(err, rows) { - queryHandler(err, rows, callback); - }); + + Step( + function getApiKey() { + self.metadataBackend.getUserMapKey(username, this); + }, + function executeQuery(err, apiKey) { + if (err) { + throw err; + } + sqlApi.query(username, apiKey, query, function(err, rows) { + queryHandler(err, rows, callback); + }); + } + ); + } -} +}; function prepareSql(sql) { diff --git a/lib/cartodb/backends/pg_connection.js b/lib/cartodb/backends/pg_connection.js new file mode 100644 index 00000000..37209b98 --- /dev/null +++ b/lib/cartodb/backends/pg_connection.js @@ -0,0 +1,96 @@ +var Step = require('step'); +var _ = require('underscore'); + +function PgConnection(metadataBackend) { + this.metadataBackend = metadataBackend; +} + +module.exports = PgConnection; + + +// Set db authentication parameters to those of the given username +// +// @param username the cartodb username, mapped to a database username +// via CartodbRedis metadata records +// +// @param params the parameters to set auth options into +// added params are: "dbuser" and "dbpassword" +// +// @param callback function(err) +// +PgConnection.prototype.setDBAuth = function(username, params, callback) { + var self = this; + + var user_params = {}; + var auth_user = global.environment.postgres_auth_user; + var auth_pass = global.environment.postgres_auth_pass; + Step( + function getId() { + self.metadataBackend.getUserId(username, this); + }, + function(err, user_id) { + if (err) throw err; + user_params['user_id'] = user_id; + var dbuser = _.template(auth_user, user_params); + _.extend(params, {dbuser:dbuser}); + + // skip looking up user_password if postgres_auth_pass + // doesn't contain the "user_password" label + if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null; + + self.metadataBackend.getUserDBPass(username, this); + }, + function(err, user_password) { + if (err) throw err; + user_params['user_password'] = user_password; + if ( auth_pass ) { + var dbpass = _.template(auth_pass, user_params); + _.extend(params, {dbpassword:dbpass}); + } + return true; + }, + function finish(err) { + callback(err); + } + ); +}; + +// Set db connection parameters to those for the given username +// +// @param dbowner cartodb username of database owner, +// mapped to a database username +// via CartodbRedis metadata records +// +// @param params the parameters to set connection options into +// added params are: "dbname", "dbhost" +// +// @param callback function(err) +// +PgConnection.prototype.setDBConn = function(dbowner, params, callback) { + var self = this; + // Add default database connection parameters + // if none given + _.defaults(params, { + dbuser: global.environment.postgres.user, + dbpassword: global.environment.postgres.password, + dbhost: global.environment.postgres.host, + dbport: global.environment.postgres.port + }); + Step( + function getConnectionParams() { + self.metadataBackend.getUserDBConnectionParams(dbowner, this); + }, + function extendParams(err, dbParams){ + if (err) throw err; + // we don't want null values or overwrite a non public user + if (params.dbuser != 'publicuser' || !dbParams.dbuser) { + delete dbParams.dbuser; + } + if ( dbParams ) _.extend(params, dbParams); + return null; + }, + function finish(err) { + callback(err); + } + ); +}; diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index a5a01620..22bbb074 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -158,7 +158,14 @@ var CartodbWindshaft = function(serverOptions) { var TemplateMapsController = require('./controllers/template_maps'), templateMapsController = new TemplateMapsController( - ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry + ws, + serverOptions, + templateMaps, + cartoData, + template_baseurl, + surrogateKeysCache, + NamedMapsCacheEntry, + serverOptions.pgConnection ); templateMapsController.register(ws); diff --git a/lib/cartodb/controllers/template_maps.js b/lib/cartodb/controllers/template_maps.js index db525c7c..5b61ae56 100644 --- a/lib/cartodb/controllers/template_maps.js +++ b/lib/cartodb/controllers/template_maps.js @@ -2,7 +2,7 @@ var Step = require('step'); var _ = require('underscore'); function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache, - NamedMapsCacheEntry) { + NamedMapsCacheEntry, pgConnection) { this.app = app; this.serverOptions = serverOptions; this.templateMaps = templateMaps; @@ -10,6 +10,7 @@ function TemplateMapsController(app, serverOptions, templateMaps, metadataBacken this.templateBaseUrl = templateBaseUrl; this.surrogateKeysCache = surrogateKeysCache; this.NamedMapsCacheEntry = NamedMapsCacheEntry; + this.pgConnection = pgConnection; } module.exports = TemplateMapsController; @@ -465,11 +466,11 @@ TemplateMapsController.prototype.setDBParams = function(cdbuser, params, callbac var self = this; Step( function setAuth() { - self.serverOptions.setDBAuth(cdbuser, params, this); + self.pgConnection.setDBAuth(cdbuser, params, this); }, function setConn(err) { if ( err ) throw err; - self.serverOptions.setDBConn(cdbuser, params, this); + self.pgConnection.setDBConn(cdbuser, params, this); }, function finish(err) { callback(err); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 59e9fc3f..ef53884c 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,6 +1,7 @@ var _ = require('underscore'); var Step = require('step'); var QueryTablesApi = require('./api/query_tables_api'); +var PgConnection = require('./backends/pg_connection'); var crypto = require('crypto'); var LZMA = require('lzma').LZMA; @@ -35,7 +36,8 @@ module.exports = function(redisPool) { var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis; var cartoData = require('cartodb-redis')(redisOpts), lzmaWorker = new LZMA(), - queryTablesApi = new QueryTablesApi(); + pgConnection = new PgConnection(cartoData), + queryTablesApi = new QueryTablesApi(pgConnection, cartoData); var rendererConfig = _.defaults(global.environment.renderer || {}, { cache_ttl: 60000, // milliseconds @@ -102,6 +104,9 @@ module.exports = function(redisPool) { // Re-use redisPool me.redis.pool = redisPool; + // Re-use pgConnection + me.pgConnection = pgConnection; + /* This whole block is about generating X-Cache-Channel { */ // TODO: review lifetime of elements of this cache @@ -211,14 +216,7 @@ module.exports = function(redisPool) { if ( req.profiler ) req.profiler.done('getSignerMapKey'); key = data; } - queryTablesApi.getAffectedTablesInQuery(user, { - user: req.params.dbuser, - pass: req.params.dbpass, - host: req.params.dbhost, - port: req.params.dbport, - dbname: req.params.dbname, - api_key: key - }, sql, this); // in addCacheChannel + queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel }, function finish(err, data) { next(err,data); @@ -329,28 +327,16 @@ module.exports = function(redisPool) { done(); }); - var sql = []; - _.each(mapconfig.layers, function(lyr) { - sql.push(lyr.options.sql); - }); - sql = sql.join(';'); + var sql = mapconfig.layers.map(function(layer) { + return layer.options.sql; + }).join(';'); var dbName = req.params.dbname; - var usr = this.userByReq(req); - var key = req.params.map_key || req.params.api_key; - var cacheKey = dbName + ':' + token; Step( function getAffectedTablesAndLastUpdatedTime() { - queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, { - user: req.params.dbuser, - pass: req.params.dbpass, - host: req.params.dbhost, - port: req.params.dbport, - dbname: req.params.dbname, - api_key: key - }, sql, this); + queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { if (req.profiler) req.profiler.done('queryTablesAndLastUpdated'); @@ -405,92 +391,6 @@ module.exports = function(redisPool) { return mat[1]; }; - // Set db authentication parameters to those of the given username - // - // @param username the cartodb username, mapped to a database username - // via CartodbRedis metadata records - // - // @param params the parameters to set auth options into - // added params are: "dbuser" and "dbpassword" - // - // @param callback function(err) - // - me.setDBAuth = function(username, params, callback) { - - var user_params = {}; - var auth_user = global.environment.postgres_auth_user; - var auth_pass = global.environment.postgres_auth_pass; - Step( - function getId() { - cartoData.getUserId(username, this); - }, - function(err, user_id) { - if (err) throw err; - user_params['user_id'] = user_id; - var dbuser = _.template(auth_user, user_params); - _.extend(params, {dbuser:dbuser}); - - // skip looking up user_password if postgres_auth_pass - // doesn't contain the "user_password" label - if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null; - - cartoData.getUserDBPass(username, this); - }, - function(err, user_password) { - if (err) throw err; - user_params['user_password'] = user_password; - if ( auth_pass ) { - var dbpass = _.template(auth_pass, user_params); - _.extend(params, {dbpassword:dbpass}); - } - return true; - }, - function finish(err) { - callback(err); - } - ); - }; - - // Set db connection parameters to those for the given username - // - // @param dbowner cartodb username of database owner, - // mapped to a database username - // via CartodbRedis metadata records - // - // @param params the parameters to set connection options into - // added params are: "dbname", "dbhost" - // - // @param callback function(err) - // - me.setDBConn = function(dbowner, params, callback) { - // Add default database connection parameters - // if none given - _.defaults(params, { - dbuser: global.environment.postgres.user, - dbpassword: global.environment.postgres.password, - dbhost: global.environment.postgres.host, - dbport: global.environment.postgres.port - }); - Step( - function getConnectionParams() { - cartoData.getUserDBConnectionParams(dbowner, this); - }, - function extendParams(err, dbParams){ - if (err) throw err; - // we don't want null values or overwrite a non public user - if (params.dbuser != 'publicuser' || !dbParams.dbuser) { - delete dbParams.dbuser; - } - if ( dbParams ) _.extend(params, dbParams); - return null; - }, - function finish(err) { - callback(err); - } - ); - }; - - // Check if a request is authorized by a signer // // @param req express request object @@ -589,7 +489,7 @@ module.exports = function(redisPool) { _.extend(req.params, { _authorizedByApiKey: true }); // authorized by api key, login as the given username and stop - that.setDBAuth(user, req.params, function(err) { + pgConnection.setDBAuth(user, req.params, function(err) { callback(err, true); // authorized (or error) }); }, @@ -624,7 +524,7 @@ module.exports = function(redisPool) { // Authorized by "signed_by" ! _.extend(req.params, { _authorizedBySigner: signed_by }); - that.setDBAuth(signed_by, req.params, function(err) { + pgConnection.setDBAuth(signed_by, req.params, function(err) { if (req.profiler) req.profiler.done('setDBAuth'); callback(err, true); // authorized (or error) }); @@ -738,7 +638,7 @@ module.exports = function(redisPool) { }, function getDatabase(err){ if(err) throw err; - that.setDBConn(user, req.params, this); + pgConnection.setDBConn(user, req.params, this); }, function getGeometryType(err){ if (req.profiler) req.profiler.done('setDBConn');