Merge pull request #204 from CartoDB/CDB-3686

Configurable QueryTablesAPI to call directly postgresql

NEWS.md

@@ -1,6 +1,10 @@
 1.15.1 -- 2014-mm-dd
 --------------------
 
+New features:
+ - Configurable QueryTablesAPI to call directly postgresql using cartodb-psql
+   or to keep using a request to the SQL API
+
 1.15.0 -- 2014-08-13
 --------------------
 Enhancements:
@@ -18,8 +22,10 @@ Enhancements:
  - SQL API requests moved to its own entity
 
 New features:
- - Affected tables and last updated time for a query are performed in a single request to the SQL API
+ - Affected tables and last updated time for a query are performed in a single
- - Allow specifying the tile format, upgrades windshaft and grainstore dependencies for this matter
+   request to the SQL API
+ - Allow specifying the tile format, upgrades windshaft and grainstore
+   dependencies for this matter
 
 
 1.13.1 -- 2014-08-04

lib/cartodb/api/query_tables_api.js

@@ -1,4 +1,5 @@
-var sqlApi = require('../sql/sql_api');
+var sqlApi = require('../sql/sql_api'),
+    PSQL   = require('cartodb-psql');
 
 function QueryTablesApi() {
 }
@@ -33,7 +34,7 @@ QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, table
     });
 };
 
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key, sql, callback) {
+QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
     // Replace mapnik tokens
     sql = sql
         .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
@@ -45,20 +46,32 @@ QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key,
     sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
 
     // call sql api
-    sqlApi.query(username, api_key, sql, function(err, rows){
+    if (shouldQueryPostgresDirectly()) {
-        if (err){
+        var psql = new PSQL(options);
-            var msg = err.message ? err.message : err;
+        psql.query(sql, function(err, resultSet) {
-            callback(new Error('could not fetch source tables: ' + msg));
+            var rows = resultSet.rows || [];
-            return;
+            handleAffectedTablesInQueryRows(err, rows, callback);
-        }
+        });
-        var qtables = rows[0].cdb_querytables;
+    } else {
-        var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+        sqlApi.query(username, options.api_key, sql, function(err, rows) {
-        tableNames = tableNames ? tableNames.split(',') : [];
+            handleAffectedTablesInQueryRows(err, rows, callback);
-        callback(null, tableNames);
+        });
-    });
+    }
 };
 
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, api_key, sql, callback) {
+function handleAffectedTablesInQueryRows(err, rows, callback) {
+    if (err){
+        var msg = err.message ? err.message : err;
+        callback(new Error('could not fetch source tables: ' + msg));
+        return;
+    }
+    var qtables = rows[0].cdb_querytables;
+    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+    tableNames = tableNames ? tableNames.split(',') : [];
+    callback(null, tableNames);
+}
+
+QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
     sql = sql
         .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
         .replace(affectedTableRegexCache.pixel_width, '1')
@@ -66,30 +79,48 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
     ;
 
     var query = [
-        'SELECT',
+        'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$' + sql + '$windshaft$) as tablenames)',
-            'CDB_QueryTables($windshaft$' + sql + '$windshaft$) as tablenames,',
+        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'EXTRACT(EPOCH FROM max(updated_at)) as max',
         'FROM CDB_TableMetadata m',
-            'WHERE m.tabname = any (CDB_QueryTables($windshaft$' + sql + '$windshaft$)::regclass[])'
+        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
     ].join(' ');
 
-    sqlApi.query(username, api_key, query, function(err, rows){
+    if (shouldQueryPostgresDirectly()) {
-        if (err || rows.length === 0) {
+        var psql = new PSQL(options);
-            var msg = err.message ? err.message : err;
+        psql.query(query, function(err, resultSet) {
-            callback(new Error('could not fetch affected tables and last updated time: ' + msg));
+            var rows = resultSet.rows || [];
-            return;
+            handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback);
-        }
-
-        var result = rows[0];
-
-        var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-        tableNames = tableNames ? tableNames.split(',') : [];
-
-        var lastUpdatedTime = result.max || 0;
-
-        callback(null, {
-            affectedTables: tableNames,
-            lastUpdatedTime: lastUpdatedTime * 1000
         });
-    });
+    } else {
+        sqlApi.query(username, options.api_key, query, function(err, rows) {
+            handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback);
+        });
+    }
 };
+
+function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
+    if (err || rows.length === 0) {
+        var msg = err.message ? err.message : err;
+        callback(new Error('could not fetch affected tables and last updated time: ' + msg));
+        return;
+    }
+
+    var result = rows[0];
+
+    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
+    tableNames = tableNames ? tableNames.split(',') : [];
+
+    var lastUpdatedTime = result.max || 0;
+
+    callback(null, {
+        affectedTables: tableNames,
+        lastUpdatedTime: lastUpdatedTime * 1000
+    });
+}
+
+
+function shouldQueryPostgresDirectly() {
+    return global.environment
+        && global.environment.enabledFeatures
+        && global.environment.enabledFeatures.cdbQueryTablesFromPostgres;
+}

lib/cartodb/server_options.js

@@ -192,7 +192,14 @@ module.exports = function(){
                 if ( req.profiler ) req.profiler.done('getSignerMapKey');
                 key = data;
               }
-              queryTablesApi.getAffectedTablesInQuery(user, key, sql, this); // in addCacheChannel
+              queryTablesApi.getAffectedTablesInQuery(user, {
+                  user: req.params.dbuser,
+                  pass: req.params.dbpass,
+                  host: req.params.dbhost,
+                  port: req.params.dbport,
+                  dbname: req.params.dbname,
+                  api_key: key
+              }, sql, this); // in addCacheChannel
             },
             function finish(err, data) {
               next(err,data);
@@ -317,7 +324,14 @@ module.exports = function(){
 
         Step(
             function getAffectedTablesAndLastUpdatedTime() {
-                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, key, sql, this);
+                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
+                    user: req.params.dbuser,
+                    pass: req.params.dbpass,
+                    host: req.params.dbhost,
+                    port: req.params.dbport,
+                    dbname: req.params.dbname,
+                    api_key: key
+                }, sql, this);
             },
             function handleAffectedTablesAndLastUpdatedTime(err, result) {
                 if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');

npm-shrinkwrap.json

@@ -937,6 +937,29 @@
       "version": "0.9.0",
       "from": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0"
     },
+    "cartodb-psql": {
+      "version": "0.3.1",
+      "from": "git://github.com/CartoDB/node-cartodb-psql.git#0.3.1",
+      "dependencies": {
+          "pg": {
+              "version": "2.6.2",
+              "dependencies": {
+                  "generic-pool": {
+                      "version": "2.0.3"
+                  },
+                  "buffer-writer": {
+                      "version": "1.0.0"
+                  }
+              }
+          },
+          "step": {
+              "version": "0.0.5"
+          },
+          "underscore": {
+              "version": "1.6.0"
+          }
+      }
+    },
     "redis-mpool": {
       "version": "0.1.0",
       "from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",

package.json

@@ -28,6 +28,7 @@
         "step": "~0.0.5",
         "request": "2.9.202",
         "cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0",
+        "cartodb-psql": "git://github.com/CartoDB/node-cartodb-psql.git#0.3.1",
         "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
         "mapnik": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1",
         "lzma": "~1.2.3",

test/acceptance/multilayer.js

@@ -23,7 +23,11 @@ serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-suite('multilayer', function() {
+[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
+
+global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
+
+suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
@@ -112,13 +116,12 @@ suite('multilayer', function() {
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
               var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join('');
-              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+              assert.equal(sentquery.q, 'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$'
                 + expectedQuery
-                + '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+                + '$windshaft$) as tablenames)'
+                + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
                 + ' FROM CDB_TableMetadata m'
-                + ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+                + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
-                + expectedQuery
-                + '$windshaft$)::regclass[])');
 
               assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL,
                 function(err, similarity) {
@@ -395,13 +398,12 @@ suite('multilayer', function() {
                   .replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)')
                   .replace(/!pixel_width!/g, '1')
                   .replace(/!pixel_height!/g, '1');
-              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+              assert.equal(sentquery.q, 'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$'
                   + expectedQuery
-                  + '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+                  + '$windshaft$) as tablenames)'
+                  + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
                   + ' FROM CDB_TableMetadata m'
-                  + ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+                  + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
-                  + expectedQuery
-                  + '$windshaft$)::regclass[])');
 
               assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err, similarity) {
@@ -433,13 +435,12 @@ suite('multilayer', function() {
                   .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
                   .replace('!pixel_width!', '1')
                   .replace('!pixel_height!', '1');
-              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+              assert.equal(sentquery.q, 'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$'
-                + expectedQuery
-                + '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
-                  + ' FROM CDB_TableMetadata m'
-                  + ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
                   + expectedQuery
-                  + '$windshaft$)::regclass[])');
+                  + '$windshaft$) as tablenames)'
+                  + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+                  + ' FROM CDB_TableMetadata m'
+                  + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
 
               assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err, similarity) {
@@ -1345,3 +1346,4 @@ suite('multilayer', function() {
     
 });
 
+});

test/acceptance/server.js

@@ -19,7 +19,12 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-suite('server', function() {
+[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
+
+global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
+
+suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
+
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
@@ -1389,3 +1394,4 @@ suite('server', function() {
     
 });
 
+});

test/acceptance/templates.js

@@ -25,7 +25,11 @@ var serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-suite('template_api', function() {
+[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
+
+global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
+
+suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
@@ -1948,3 +1952,4 @@ suite('template_api', function() {
     
 });
 
+});