QueryTables and last updated_at retrieved with user

Move setDBAuth and setDBConn to PgConnection entity
 - It uses cartodb-redis to retrieve datasource configuration
Start using it in ServerOptions, TemplateMaps and QueryTablesApi
QueryTablesApi don't receive anymore the connection/credentials
 - It will always use an authenticated query to retrieve last update
 - That will allow to query affected private tables last update

lib/cartodb/api/query_tables_api.js

@@ -1,7 +1,10 @@
-var sqlApi = require('../sql/sql_api'),
+var sqlApi = require('../sql/sql_api');
-    PSQL   = require('cartodb-psql');
+var PSQL = require('cartodb-psql');
+var Step = require('step');
 
-function QueryTablesApi() {
+function QueryTablesApi(pgConnection, metadataBackend) {
+    this.pgConnection = pgConnection;
+    this.metadataBackend = metadataBackend;
 }
 
 var affectedTableRegexCache = {
@@ -14,11 +17,11 @@ var affectedTableRegexCache = {
 module.exports = QueryTablesApi;
 
 
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
+QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
 
     var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
 
-    runQuery(username, options, query, handleAffectedTablesInQueryRows, callback);
+    this.runQuery(username, query, handleAffectedTablesInQueryRows, callback);
 };
 
 function handleAffectedTablesInQueryRows(err, rows, callback) {
@@ -33,7 +36,7 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
     callback(null, tableNames);
 }
 
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
+QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
 
     var query = [
         'WITH querytables AS (',
@@ -44,7 +47,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
         'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
     ].join(' ');
 
-    runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
+    this.runQuery(username, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
 };
 
 function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
@@ -68,20 +71,60 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
 }
 
 
-function runQuery(username, options, query, queryHandler, callback) {
+QueryTablesApi.prototype.runQuery = function(username, query, queryHandler, callback) {
+    var self = this;
+
     if (shouldQueryPostgresDirectly()) {
-        var psql = new PSQL(options);
+
+        var params = {};
+
+        Step(
+            function setAuth() {
+                self.pgConnection.setDBAuth(username, params, this);
+            },
+            function setConn(err) {
+                if (err) {
+                    throw err;
+                }
+                self.pgConnection.setDBConn(username, params, this);
+            },
+            function executeQuery(err) {
+                if (err) {
+                    throw err;
+                }
+                var psql = new PSQL({
+                    user: params.dbuser,
+                    pass: params.dbpass,
+                    host: params.dbhost,
+                    port: params.dbport,
+                    dbname: params.dbname
+                });
                 psql.query(query, function(err, resultSet) {
                     resultSet = resultSet || {};
                     var rows = resultSet.rows || [];
                     queryHandler(err, rows, callback);
                 });
+            }
+        );
+
     } else {
-        sqlApi.query(username, options.api_key, query, function(err, rows) {
+
+        Step(
+            function getApiKey() {
+                self.metadataBackend.getUserMapKey(username, this);
+            },
+            function executeQuery(err, apiKey) {
+                if (err) {
+                    throw err;
+                }
+                sqlApi.query(username, apiKey, query, function(err, rows) {
                     queryHandler(err, rows, callback);
                 });
             }
-}
+        );
+
+    }
+};
 
 
 function prepareSql(sql) {

lib/cartodb/backends/pg_connection.js

@@ -0,0 +1,96 @@
+var Step = require('step');
+var _ = require('underscore');
+
+function PgConnection(metadataBackend) {
+    this.metadataBackend = metadataBackend;
+}
+
+module.exports = PgConnection;
+
+
+// Set db authentication parameters to those of the given username
+//
+// @param username the cartodb username, mapped to a database username
+//                 via CartodbRedis metadata records
+//
+// @param params the parameters to set auth options into
+//               added params are: "dbuser" and "dbpassword"
+//
+// @param callback function(err)
+//
+PgConnection.prototype.setDBAuth = function(username, params, callback) {
+    var self = this;
+
+    var user_params = {};
+    var auth_user = global.environment.postgres_auth_user;
+    var auth_pass = global.environment.postgres_auth_pass;
+    Step(
+        function getId() {
+            self.metadataBackend.getUserId(username, this);
+        },
+        function(err, user_id) {
+            if (err) throw err;
+            user_params['user_id'] = user_id;
+            var dbuser = _.template(auth_user, user_params);
+            _.extend(params, {dbuser:dbuser});
+
+            // skip looking up user_password if postgres_auth_pass
+            // doesn't contain the "user_password" label
+            if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
+
+            self.metadataBackend.getUserDBPass(username, this);
+        },
+        function(err, user_password) {
+            if (err) throw err;
+            user_params['user_password'] = user_password;
+            if ( auth_pass ) {
+                var dbpass = _.template(auth_pass, user_params);
+                _.extend(params, {dbpassword:dbpass});
+            }
+            return true;
+        },
+        function finish(err) {
+            callback(err);
+        }
+    );
+};
+
+// Set db connection parameters to those for the given username
+//
+// @param dbowner cartodb username of database owner,
+//                mapped to a database username
+//                via CartodbRedis metadata records
+//
+// @param params the parameters to set connection options into
+//               added params are: "dbname", "dbhost"
+//
+// @param callback function(err)
+//
+PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
+    var self = this;
+    // Add default database connection parameters
+    // if none given
+    _.defaults(params, {
+        dbuser: global.environment.postgres.user,
+        dbpassword: global.environment.postgres.password,
+        dbhost: global.environment.postgres.host,
+        dbport: global.environment.postgres.port
+    });
+    Step(
+        function getConnectionParams() {
+            self.metadataBackend.getUserDBConnectionParams(dbowner, this);
+        },
+        function extendParams(err, dbParams){
+            if (err) throw err;
+            // we don't want null values or overwrite a non public user
+            if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
+                delete dbParams.dbuser;
+            }
+            if ( dbParams ) _.extend(params, dbParams);
+            return null;
+        },
+        function finish(err) {
+            callback(err);
+        }
+    );
+};

lib/cartodb/cartodb_windshaft.js

@@ -158,7 +158,14 @@ var CartodbWindshaft = function(serverOptions) {
 
     var TemplateMapsController = require('./controllers/template_maps'),
         templateMapsController = new TemplateMapsController(
-            ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry
+            ws,
+            serverOptions,
+            templateMaps,
+            cartoData,
+            template_baseurl,
+            surrogateKeysCache,
+            NamedMapsCacheEntry,
+            serverOptions.pgConnection
         );
     templateMapsController.register(ws);
 

lib/cartodb/controllers/template_maps.js

@@ -2,7 +2,7 @@ var Step = require('step');
 var _ = require('underscore');
 
 function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache,
-                                NamedMapsCacheEntry) {
+                                NamedMapsCacheEntry, pgConnection) {
     this.app = app;
     this.serverOptions = serverOptions;
     this.templateMaps = templateMaps;
@@ -10,6 +10,7 @@ function TemplateMapsController(app, serverOptions, templateMaps, metadataBacken
     this.templateBaseUrl = templateBaseUrl;
     this.surrogateKeysCache = surrogateKeysCache;
     this.NamedMapsCacheEntry = NamedMapsCacheEntry;
+    this.pgConnection = pgConnection;
 }
 
 module.exports = TemplateMapsController;
@@ -465,11 +466,11 @@ TemplateMapsController.prototype.setDBParams = function(cdbuser, params, callbac
     var self = this;
     Step(
         function setAuth() {
-            self.serverOptions.setDBAuth(cdbuser, params, this);
+            self.pgConnection.setDBAuth(cdbuser, params, this);
         },
         function setConn(err) {
             if ( err ) throw err;
-            self.serverOptions.setDBConn(cdbuser, params, this);
+            self.pgConnection.setDBConn(cdbuser, params, this);
         },
         function finish(err) {
             callback(err);

lib/cartodb/server_options.js

@@ -1,6 +1,7 @@
 var _ = require('underscore');
 var Step = require('step');
 var QueryTablesApi = require('./api/query_tables_api');
+var PgConnection = require('./backends/pg_connection');
 var crypto = require('crypto');
 var LZMA = require('lzma').LZMA;
 
@@ -35,7 +36,8 @@ module.exports = function(redisPool) {
     var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis;
     var cartoData = require('cartodb-redis')(redisOpts),
         lzmaWorker = new LZMA(),
-        queryTablesApi = new QueryTablesApi();
+        pgConnection = new PgConnection(cartoData),
+        queryTablesApi = new QueryTablesApi(pgConnection, cartoData);
 
     var rendererConfig = _.defaults(global.environment.renderer || {}, {
         cache_ttl: 60000, // milliseconds
@@ -102,6 +104,9 @@ module.exports = function(redisPool) {
     // Re-use redisPool
     me.redis.pool = redisPool;
 
+    // Re-use pgConnection
+    me.pgConnection = pgConnection;
+
 /* This whole block is about generating X-Cache-Channel { */
 
     // TODO: review lifetime of elements of this cache
@@ -211,14 +216,7 @@ module.exports = function(redisPool) {
                 if ( req.profiler ) req.profiler.done('getSignerMapKey');
                 key = data;
               }
-              queryTablesApi.getAffectedTablesInQuery(user, {
+              queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
-                  user: req.params.dbuser,
-                  pass: req.params.dbpass,
-                  host: req.params.dbhost,
-                  port: req.params.dbport,
-                  dbname: req.params.dbname,
-                  api_key: key
-              }, sql, this); // in addCacheChannel
             },
             function finish(err, data) {
               next(err,data);
@@ -329,28 +327,16 @@ module.exports = function(redisPool) {
           done();
         });
 
-        var sql = [];
+        var sql = mapconfig.layers.map(function(layer) {
-        _.each(mapconfig.layers, function(lyr) {
+            return layer.options.sql;
-          sql.push(lyr.options.sql);
+        }).join(';');
-        });
-        sql = sql.join(';');
 
         var dbName = req.params.dbname;
-        var usr    = this.userByReq(req);
-        var key    = req.params.map_key || req.params.api_key;
-
         var cacheKey = dbName + ':' + token;
 
         Step(
             function getAffectedTablesAndLastUpdatedTime() {
-                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
+                queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
-                    user: req.params.dbuser,
-                    pass: req.params.dbpass,
-                    host: req.params.dbhost,
-                    port: req.params.dbport,
-                    dbname: req.params.dbname,
-                    api_key: key
-                }, sql, this);
             },
             function handleAffectedTablesAndLastUpdatedTime(err, result) {
                 if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
@@ -405,92 +391,6 @@ module.exports = function(redisPool) {
       return mat[1];
     };
 
-    // Set db authentication parameters to those of the given username
-    //
-    // @param username the cartodb username, mapped to a database username
-    //                 via CartodbRedis metadata records
-    //
-    // @param params the parameters to set auth options into
-    //               added params are: "dbuser" and "dbpassword"
-    //
-    // @param callback function(err) 
-    //
-    me.setDBAuth = function(username, params, callback) {
-
-      var user_params = {};
-      var auth_user = global.environment.postgres_auth_user;
-      var auth_pass = global.environment.postgres_auth_pass;
-      Step(
-        function getId() {
-          cartoData.getUserId(username, this);
-        },
-        function(err, user_id) {
-          if (err) throw err;
-          user_params['user_id'] = user_id;
-          var dbuser = _.template(auth_user, user_params);
-          _.extend(params, {dbuser:dbuser});
-
-          // skip looking up user_password if postgres_auth_pass
-          // doesn't contain the "user_password" label 
-          if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
-
-          cartoData.getUserDBPass(username, this);
-        },
-        function(err, user_password) {
-          if (err) throw err;
-          user_params['user_password'] = user_password;
-          if ( auth_pass ) {
-            var dbpass = _.template(auth_pass, user_params);
-            _.extend(params, {dbpassword:dbpass});
-          }
-          return true;
-        },
-        function finish(err) {
-          callback(err); 
-        }
-      );
-    };
-
-    // Set db connection parameters to those for the given username
-    //
-    // @param dbowner cartodb username of database owner,
-    //                mapped to a database username
-    //                via CartodbRedis metadata records
-    //
-    // @param params the parameters to set connection options into
-    //               added params are: "dbname", "dbhost"
-    //
-    // @param callback function(err) 
-    //
-    me.setDBConn = function(dbowner, params, callback) {
-      // Add default database connection parameters
-      // if none given
-      _.defaults(params, {
-        dbuser: global.environment.postgres.user,
-        dbpassword: global.environment.postgres.password,
-        dbhost: global.environment.postgres.host,
-        dbport: global.environment.postgres.port
-      });
-      Step(
-        function getConnectionParams() {
-            cartoData.getUserDBConnectionParams(dbowner, this);
-        },
-        function extendParams(err, dbParams){
-            if (err) throw err;
-            // we don't want null values or overwrite a non public user
-            if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
-                delete dbParams.dbuser;
-            }
-            if ( dbParams ) _.extend(params, dbParams);
-            return null;
-        },
-        function finish(err) {
-          callback(err);
-        }
-      );
-    };
-
-
     // Check if a request is authorized by a signer
     //
     // @param req express request object
@@ -589,7 +489,7 @@ module.exports = function(redisPool) {
                 _.extend(req.params, { _authorizedByApiKey: true });
 
                 // authorized by api key, login as the given username and stop
-                that.setDBAuth(user, req.params, function(err) {
+                pgConnection.setDBAuth(user, req.params, function(err) {
                   callback(err, true); // authorized (or error)
                 });
             },
@@ -624,7 +524,7 @@ module.exports = function(redisPool) {
 
                 // Authorized by "signed_by" !
                 _.extend(req.params, { _authorizedBySigner: signed_by });
-                that.setDBAuth(signed_by, req.params, function(err) {
+                pgConnection.setDBAuth(signed_by, req.params, function(err) {
                   if (req.profiler) req.profiler.done('setDBAuth');
                   callback(err, true); // authorized (or error)
                 });
@@ -738,7 +638,7 @@ module.exports = function(redisPool) {
             },
             function getDatabase(err){
                 if(err) throw err;
-                that.setDBConn(user, req.params, this);
+                pgConnection.setDBConn(user, req.params, this);
             },
             function getGeometryType(err){
                 if (req.profiler) req.profiler.done('setDBConn');