Add support for CTE in sql windowing, add unit tests

This is still an undocumented feature, but as long as it's present
and used (by cartodb UI) better tested than broken...

NOTE: more tests are needed for CTE and RETURNING queries

NEWS.md

@@ -2,7 +2,7 @@
 -----
 * Add "fields" member in JSON return (#97)
 * Add --skipfields switch to cdbsql
-* Fix windowing with non-uppercased SELECTs
+* Fix windowing with CTE
 
 1.4.0
 -----

app/controllers/app.js

@@ -77,18 +77,6 @@ function sanitize_filename(filename) {
   return filename;
 }
 
-// little hack for UI
-//
-// TODO:drop, fix in the UI (it's not documented in doc/API)
-//
-function window_sql (sql, limit, offset) {
-    // only window select functions (NOTE: "values" will be broken, "with" will be broken)
-    if (_.isNumber(limit) && _.isNumber(offset) && sql.match(/^\s*SELECT\s/i) ) {
-        return "SELECT * FROM (" + sql + ") AS cdbq_1 LIMIT " + limit + " OFFSET " + offset;
-    } 
-    return sql;
-}
-
 // request handlers
 function handleQuery(req, res) {
 
@@ -267,7 +255,7 @@ function handleQuery(req, res) {
                 if (err) throw err;
 
                 // TODO: drop this, fix UI!
-                sql = window_sql(sql,limit,offset);
+                sql = PSQL.window_sql(sql,limit,offset);
 
                 var opts = {
                   sink: res,

app/models/psql.js

@@ -132,4 +132,67 @@ var PSQL = function(user_id, db) {
     return me;
 };
 
+// little hack for UI
+//
+// TODO:drop, fix in the UI (it's not documented in doc/API)
+//
+PSQL.window_sql = function(sql, limit, offset) {
+  // only window select functions (NOTE: "values" will be broken, "with" will be broken)
+  if (!_.isNumber(limit) || !_.isNumber(offset) ) return sql;
+
+  var cte = '';
+
+  if ( sql.match(/^\s*WITH\s/i) ) {
+
+    var rem = sql; // analyzed portion of sql
+    var q; // quote char
+    var n = 0; // nested parens level
+    var s = 0; // 0:outQuote, 1:inQuote
+    var l;
+    while (1) {
+      l = rem.search(/[("')]/);
+//console.log("REM Is " + rem);
+      if ( l < 0 ) {
+        console.log("Malformed SQL");
+        return sql;
+      }
+      var f = rem.charAt(l);
+//console.log("n:" + n + " s:" + s + " l:" + l + " charAt(l):" + f + " charAt(l+1):" + rem.charAt(l+1));
+      if ( s == 0 ) {
+        if ( f == '(' ) ++n; 
+        else if ( f == ')' ) {
+          if ( ! --n ) { // end of CTE
+            cte += rem.substr(0, l+1);
+            rem = rem.substr(l+1);
+  //console.log("Out of cte, rem is " + rem);
+            if ( rem.search(/^s*,/) < 0 ) break;
+            else continue; // cte and rem already updated
+          }
+        }
+        else { // enter quoting
+          s = 1; q = f;
+        }
+      }
+      else if ( f == q ) {
+        if ( rem.charAt(l+1) == f ) ++l; // escaped
+        else s = 0; // exit quoting
+      }
+      cte += rem.substr(0, l+1);
+      rem = rem.substr(l+1);
+    }
+/*
+console.log("cte: " + cte);
+console.log("rem: " + rem);
+*/
+    sql = rem; //sql.substr(l+1);
+  }
+
+
+  if ( sql.match(/^\s*SELECT\s/i) ) {
+      return cte + "SELECT * FROM (" + sql + ") AS cdbq_1 LIMIT " + limit + " OFFSET " + offset;
+  } 
+
+  return cte + sql;
+}
+
 module.exports = PSQL;

test/unit/psql.test.js

@@ -101,4 +101,38 @@ test('test publicuser cannot execute INSERT on db', function(done){
   });
 });
 
+test('Windowed SQL with simple select', function(){
+  // NOTE: intentionally mixed-case and space-padded
+  var sql = "\n \tSEleCT * from table1";
+  var out = PSQL.window_sql(sql, 1, 0);
+  assert.equal(out, "SELECT * FROM (" + sql + ") AS cdbq_1 LIMIT 1 OFFSET 0");
+});
+
+test('Windowed SQL with CTE select', function(){
+  // NOTE: intentionally mixed-case and space-padded
+  var cte = "\n \twiTh  x as( update test set x=x+1)";
+  var select = "\n \tSEleCT * from x";
+  var sql = cte + select;
+  var out = PSQL.window_sql(sql, 1, 0);
+  assert.equal(out, cte + "SELECT * FROM (" + select + ") AS cdbq_1 LIMIT 1 OFFSET 0");
+});
+
+test('Windowed SQL with CTE update', function(){
+  // NOTE: intentionally mixed-case and space-padded
+  var cte = "\n \twiTh  a as( update test set x=x+1)";
+  var upd = "\n \tupdate tost set y=x from x";
+  var sql = cte + upd;
+  var out = PSQL.window_sql(sql, 1, 0);
+  assert.equal(out, sql);
+});
+
+test('Windowed SQL with complex CTE and insane quoting', function(){
+  // NOTE: intentionally mixed-case and space-padded
+  var cte = "\n \twiTh \"('a\" as( update \"\"\"test)\" set x='x'+1), \")b(\" as ( select ')))\"' from z )";
+  var sel = "\n \tselect '\"' from x";
+  var sql = cte + sel;
+  var out = PSQL.window_sql(sql, 1, 0);
+  assert.equal(out, cte + "SELECT * FROM (" + sel + ") AS cdbq_1 LIMIT 1 OFFSET 0");
+});
+
 });