From d54e2f5a073ac466b83021c4477c19d1d66499cf Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 12 Apr 2018 12:25:28 -0700
Subject: [PATCH 001/133] Implementation including multer, custom storage
 engine, and pg-copy, but without turning over pg-copy, and demonstrating the
 missing 'sql' parameter in the custom storage engine.

---
 app/controllers/copy_controller.js | 98 ++++++++++++++++++++++++++++++
 app/middlewares/body-parser.js     |  4 +-
 app/server.js                      | 10 +++
 app/utils/multer-pg-copy.js        | 83 +++++++++++++++++++++++++
 package.json                       |  1 +
 5 files changed, 194 insertions(+), 2 deletions(-)
 create mode 100644 app/controllers/copy_controller.js
 create mode 100644 app/utils/multer-pg-copy.js

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
new file mode 100644
index 00000000..c3bf6578
--- /dev/null
+++ b/app/controllers/copy_controller.js
@@ -0,0 +1,98 @@
+'use strict';
+
+var _ = require('underscore');
+var step = require('step');
+var assert = require('assert');
+var PSQL = require('cartodb-psql');
+var CachedQueryTables = require('../services/cached-query-tables');
+var queryMayWrite = require('../utils/query_may_write');
+
+var formats = require('../models/formats');
+
+var sanitize_filename = require('../utils/filename_sanitizer');
+var getContentDisposition = require('../utils/content_disposition');
+const userMiddleware = require('../middlewares/user');
+const errorMiddleware = require('../middlewares/error');
+const authorizationMiddleware = require('../middlewares/authorization');
+const connectionParamsMiddleware = require('../middlewares/connection-params');
+const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
+const { initializeProfilerMiddleware } = require('../middlewares/profiler');
+const rateLimitsMiddleware = require('../middlewares/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
+
+var ONE_YEAR_IN_SECONDS = 31536000; // 1 year time to live by default
+
+// We need NPM body-parser so we can use the multer and
+// still decode the urlencoded 'sql' parameter from
+// the POST body
+var bodyParser = require('body-parser'); // NPM body-parser
+
+// We need multer to support multi-part POST content
+var multer = require('multer');
+
+// The default multer storage engines (file/memory) don't
+// do what we need, which is pipe the multer read stream
+// straight into the pg-copy write stream, so we use
+// a custom storage engine
+var multerpgcopy = require('../utils/multer-pg-copy');
+var upload = multer({ storage: multerpgcopy() });
+
+// var upload = multer({ dest: '/tmp/' });
+
+function CopyController(metadataBackend, userDatabaseService, tableCache, statsd_client, userLimitsService) {
+    this.metadataBackend = metadataBackend;
+    this.statsd_client = statsd_client;
+    this.userDatabaseService = userDatabaseService;
+    this.queryTables = new CachedQueryTables(tableCache);
+    this.userLimitsService = userLimitsService;
+}
+
+CopyController.prototype.route = function (app) {
+    const { base_url } = global.settings;
+    const copyFromMiddlewares = endpointGroup => {
+        return [
+            initializeProfilerMiddleware('query'),
+            userMiddleware(),
+            rateLimitsMiddleware(this.userLimitsService, endpointGroup),
+            authorizationMiddleware(this.metadataBackend),
+            connectionParamsMiddleware(this.userDatabaseService),
+            timeoutLimitsMiddleware(this.metadataBackend),
+            // bodyParser.urlencoded({ extended: true }),
+            upload.single('file'),
+            this.handleCopyFrom.bind(this),
+            errorMiddleware()
+        ];
+    };
+
+    app.post(`${base_url}/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
+};
+
+// jshint maxcomplexity:21
+CopyController.prototype.handleCopyFrom = function (req, res, next) {
+
+    // Why doesn't this function do much of anything?
+    // Because all the excitement is in the bodyParser and the upload
+    // middlewards, the first of which should fill out the body.params.sql
+    // statement and the second of which should run that statement and 
+    // upload it into pgsql.
+    // All that's left here, is to read the number of records inserted
+    // and to return some information to the caller on what exactly 
+    // happened.
+    // Test with:
+    // curl --form file=@package.json --form sql="COPY this FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom
+
+    req.aborted = false;
+    req.on("close", function() {
+        if (req.formatter && _.isFunction(req.formatter.cancel)) {
+            req.formatter.cancel();
+        }
+        req.aborted = true; // TODO: there must be a builtin way to check this
+    });
+
+    console.debug("CopyController.prototype.handleCopyFrom: sql = '%s'", req.body.sql)
+
+    res.send('got into handleCopyFrom');
+        
+};
+
+module.exports = CopyController;
diff --git a/app/middlewares/body-parser.js b/app/middlewares/body-parser.js
index 6c44e295..8bd5ff87 100644
--- a/app/middlewares/body-parser.js
+++ b/app/middlewares/body-parser.js
@@ -141,5 +141,5 @@ exports.parse['application/json'] = function(req, options, fn){
     });
 };
 
-var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
-exports.parse['multipart/form-data'] = multipartMiddleware.none();
+// var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
+// exports.parse['multipart/form-data'] = multipartMiddleware.none();
diff --git a/app/server.js b/app/server.js
index 6223d4eb..ee262262 100644
--- a/app/server.js
+++ b/app/server.js
@@ -36,6 +36,7 @@ var cors = require('./middlewares/cors');
 
 var GenericController = require('./controllers/generic_controller');
 var QueryController = require('./controllers/query_controller');
+var CopyController = require('./controllers/copy_controller');
 var JobController = require('./controllers/job_controller');
 var CacheStatusController = require('./controllers/cache_status_controller');
 var HealthCheckController = require('./controllers/health_check_controller');
@@ -163,6 +164,15 @@ function App(statsClient) {
     );
     queryController.route(app);
 
+    var copyController = new CopyController(
+        metadataBackend, 
+        userDatabaseService, 
+        tableCache, 
+        statsClient, 
+        userLimitsService
+    );
+    copyController.route(app);
+    
     var jobController = new JobController(
         metadataBackend, 
         userDatabaseService, 
diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
new file mode 100644
index 00000000..f645a652
--- /dev/null
+++ b/app/utils/multer-pg-copy.js
@@ -0,0 +1,83 @@
+// This is a multer "custom storage engine", see
+// https://github.com/expressjs/multer/blob/master/StorageEngine.md
+// for the contract. 
+
+var _ = require('underscore');
+var fs = require('fs');
+var copyFrom = require('pg-copy-streams').from;
+
+var opts;
+
+function PgCopyCustomStorage (opts) {
+  this.opts = opts || {};
+}
+
+PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb) {
+
+  // Skip the pg-copy for now, just write to /tmp/
+  // so we can see what parameters are making it into 
+  // this storage handler
+  var debug_customstorage = true; 
+
+  // Hopefully the body-parser has extracted the 'sql' parameter
+  // Otherwise, this will be a short trip, as we won't be able
+  // to run the pg-copy-streams
+  var sql = req.body.sql;
+  sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
+
+  console.debug("PgCopyCustomStorage.prototype._handleFile");
+  console.debug("PgCopyCustomStorage.prototype._handleFile: sql = '%s'", sql);
+  
+  if (debug_customstorage) {
+    var outStream = fs.createWriteStream('/tmp/sqlApiUploadExample');
+    file.stream.pipe(outStream);
+    outStream.on('error', cb);
+    outStream.on('finish', function () {
+      cb(null, {
+        path: file.path,
+        size: outStream.bytesWritten
+      });
+    });
+    
+  } else {
+    // TODO, handle this nicely
+    if(!_.isString(sql)) {
+      throw new Error("sql is not set");
+    }
+
+    // We expect the pg-connect middleware to have 
+    // set this by the time we are called via multer
+    if (!req.authDbConnection) {
+      throw new Error("req.authDbConnection is not set");
+    }
+    var sessionPg = req.authDbConnection;
+
+    sessionPg.connect(function(err, client, done) {
+      if (err) {
+        return cb(err);
+      }
+
+      console.debug("XXX pg.connect");
+
+      // This is the magic part, see 
+      // https://github.com/brianc/node-pg-copy-streams
+      var outStream = client.query(copyFrom(sql), function(err, result) {
+        done(err);
+        return cb(err, result);
+      });
+
+      file.stream.on('error', cb);
+      outStream.on('error', cb);
+      outStream.on('end', cb);
+      file.stream.pipe(outStream);
+    });
+  }
+}
+
+PgCopyCustomStorage.prototype._removeFile = function _removeFile (req, file, cb) {
+  fs.unlink(file.path, cb)
+}
+
+module.exports = function (opts) {
+  return new PgCopyCustomStorage(opts)
+}
\ No newline at end of file
diff --git a/package.json b/package.json
index d2758511..5cef5e9a 100644
--- a/package.json
+++ b/package.json
@@ -31,6 +31,7 @@
     "node-statsd": "~0.0.7",
     "node-uuid": "^1.4.7",
     "oauth-client": "0.3.0",
+    "pg-copy-streams": "^1.2.0",
     "qs": "~6.2.1",
     "queue-async": "~1.0.7",
     "redis-mpool": "0.5.0",

From 0161696627fd6dbcf5815531d83f4919f1eda107 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey+github@cleverelephant.ca>
Date: Fri, 13 Apr 2018 05:43:23 -0700
Subject: [PATCH 002/133] Update copy_controller.js

---
 app/controllers/copy_controller.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index c3bf6578..88d8fdd9 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -57,7 +57,7 @@ CopyController.prototype.route = function (app) {
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
-            // bodyParser.urlencoded({ extended: true }),
+            bodyParser.urlencoded({ extended: true }),
             upload.single('file'),
             this.handleCopyFrom.bind(this),
             errorMiddleware()

From d4fef1c2b181b77f33ca7da02dac085b9af2139d Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Fri, 13 Apr 2018 16:12:01 +0200
Subject: [PATCH 003/133] Fix dependencies

---
 npm-shrinkwrap.json | 152 ++++++++++++++++++++++++++++++++++++--------
 package.json        |   1 +
 2 files changed, 126 insertions(+), 27 deletions(-)

diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index d60194df..587d750d 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -1,6 +1,6 @@
 {
   "name": "cartodb_sql_api",
-  "version": "1.48.2",
+  "version": "2.0.1",
   "dependencies": {
     "accepts": {
       "version": "1.2.13",
@@ -53,9 +53,9 @@
       "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
     },
     "aws4": {
-      "version": "1.6.0",
+      "version": "1.7.0",
       "from": "aws4@>=1.2.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz"
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz"
     },
     "balanced-match": {
       "version": "1.0.0",
@@ -110,6 +110,23 @@
       "from": "bluebird@>=3.3.3 <4.0.0",
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz"
     },
+    "body-parser": {
+      "version": "1.18.2",
+      "from": "body-parser@1.18.2",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "from": "debug@2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz"
+        },
+        "qs": {
+          "version": "6.5.1",
+          "from": "qs@6.5.1",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz"
+        }
+      }
+    },
     "boom": {
       "version": "2.10.1",
       "from": "boom@>=2.0.0 <3.0.0",
@@ -152,6 +169,11 @@
         }
       }
     },
+    "bytes": {
+      "version": "3.0.0",
+      "from": "bytes@3.0.0",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz"
+    },
     "camelcase": {
       "version": "3.0.0",
       "from": "camelcase@>=3.0.0 <4.0.0",
@@ -218,14 +240,14 @@
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
         },
         "readable-stream": {
-          "version": "2.3.5",
+          "version": "2.3.6",
           "from": "readable-stream@>=2.2.2 <3.0.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.5.tgz"
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz"
         },
         "string_decoder": {
-          "version": "1.0.3",
-          "from": "string_decoder@>=1.0.3 <1.1.0",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz"
+          "version": "1.1.1",
+          "from": "string_decoder@>=1.1.1 <1.2.0",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz"
         }
       }
     },
@@ -236,7 +258,7 @@
     },
     "content-type": {
       "version": "1.0.4",
-      "from": "content-type@>=1.0.1 <1.1.0",
+      "from": "content-type@>=1.0.4 <1.1.0",
       "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz"
     },
     "cookie": {
@@ -274,7 +296,14 @@
     "debug": {
       "version": "2.2.0",
       "from": "debug@2.2.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz"
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
+      "dependencies": {
+        "ms": {
+          "version": "0.7.1",
+          "from": "ms@0.7.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+        }
+      }
     },
     "decamelize": {
       "version": "1.2.0",
@@ -288,7 +317,7 @@
     },
     "depd": {
       "version": "1.1.2",
-      "from": "depd@>=1.1.0 <1.2.0",
+      "from": "depd@>=1.1.1 <1.2.0",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz"
     },
     "destroy": {
@@ -481,15 +510,20 @@
       "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.6.0.tgz"
     },
     "http-errors": {
-      "version": "1.3.1",
-      "from": "http-errors@>=1.3.1 <1.4.0",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz"
+      "version": "1.6.3",
+      "from": "http-errors@>=1.6.2 <1.7.0",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz"
     },
     "http-signature": {
       "version": "1.1.1",
       "from": "http-signature@>=1.1.0 <1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz"
     },
+    "iconv-lite": {
+      "version": "0.4.19",
+      "from": "iconv-lite@0.4.19",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz"
+    },
     "inflight": {
       "version": "1.0.6",
       "from": "inflight@>=1.0.4 <2.0.0",
@@ -497,7 +531,7 @@
     },
     "inherits": {
       "version": "2.0.3",
-      "from": "inherits@>=2.0.0 <3.0.0",
+      "from": "inherits@2.0.3",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
     },
     "invert-kv": {
@@ -662,7 +696,7 @@
     },
     "mime-types": {
       "version": "2.1.18",
-      "from": "mime-types@>=2.1.6 <2.2.0",
+      "from": "mime-types@>=2.1.18 <2.2.0",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz"
     },
     "minimatch": {
@@ -681,15 +715,15 @@
       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz"
     },
     "moment": {
-      "version": "2.21.0",
+      "version": "2.22.0",
       "from": "moment@>=2.10.6 <3.0.0",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.0.tgz",
       "optional": true
     },
     "ms": {
-      "version": "0.7.1",
-      "from": "ms@0.7.1",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+      "version": "2.0.0",
+      "from": "ms@2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz"
     },
     "multer": {
       "version": "1.2.1",
@@ -832,6 +866,11 @@
       "from": "pg-connection-string@0.1.3",
       "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz"
     },
+    "pg-copy-streams": {
+      "version": "1.2.0",
+      "from": "pg-copy-streams@>=1.2.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-1.2.0.tgz"
+    },
     "pg-int8": {
       "version": "1.0.1",
       "from": "pg-int8@1.0.1",
@@ -917,6 +956,28 @@
       "from": "range-parser@>=1.0.3 <1.1.0",
       "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
     },
+    "raw-body": {
+      "version": "2.3.2",
+      "from": "raw-body@2.3.2",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz",
+      "dependencies": {
+        "depd": {
+          "version": "1.1.1",
+          "from": "depd@1.1.1",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz"
+        },
+        "http-errors": {
+          "version": "1.6.2",
+          "from": "http-errors@1.6.2",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz"
+        },
+        "setprototypeof": {
+          "version": "1.0.3",
+          "from": "setprototypeof@1.0.3",
+          "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz"
+        }
+      }
+    },
     "read-pkg": {
       "version": "1.1.0",
       "from": "read-pkg@>=1.0.0 <2.0.0",
@@ -1004,17 +1065,49 @@
     "send": {
       "version": "0.13.1",
       "from": "send@0.13.1",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz"
+      "resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz",
+      "dependencies": {
+        "http-errors": {
+          "version": "1.3.1",
+          "from": "http-errors@>=1.3.1 <1.4.0",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz"
+        },
+        "ms": {
+          "version": "0.7.1",
+          "from": "ms@0.7.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+        },
+        "statuses": {
+          "version": "1.2.1",
+          "from": "statuses@>=1.2.1 <1.3.0",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+        }
+      }
     },
     "serve-static": {
       "version": "1.10.3",
       "from": "serve-static@>=1.10.2 <1.11.0",
       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz",
       "dependencies": {
+        "http-errors": {
+          "version": "1.3.1",
+          "from": "http-errors@>=1.3.1 <1.4.0",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz"
+        },
+        "ms": {
+          "version": "0.7.1",
+          "from": "ms@0.7.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+        },
         "send": {
           "version": "0.13.2",
           "from": "send@0.13.2",
           "resolved": "https://registry.npmjs.org/send/-/send-0.13.2.tgz"
+        },
+        "statuses": {
+          "version": "1.2.1",
+          "from": "statuses@>=1.2.1 <1.3.0",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
         }
       }
     },
@@ -1023,6 +1116,11 @@
       "from": "set-blocking@>=2.0.0 <3.0.0",
       "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz"
     },
+    "setprototypeof": {
+      "version": "1.1.0",
+      "from": "setprototypeof@1.1.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz"
+    },
     "sntp": {
       "version": "1.0.9",
       "from": "sntp@>=1.0.0 <2.0.0",
@@ -1066,9 +1164,9 @@
       }
     },
     "statuses": {
-      "version": "1.2.1",
-      "from": "statuses@>=1.2.1 <1.3.0",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+      "version": "1.5.0",
+      "from": "statuses@>=1.4.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz"
     },
     "step": {
       "version": "0.0.6",
@@ -1143,7 +1241,7 @@
     },
     "type-is": {
       "version": "1.6.16",
-      "from": "type-is@>=1.6.6 <1.7.0",
+      "from": "type-is@>=1.6.15 <1.7.0",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz"
     },
     "typedarray": {
@@ -1158,7 +1256,7 @@
     },
     "unpipe": {
       "version": "1.0.0",
-      "from": "unpipe@>=1.0.0 <1.1.0",
+      "from": "unpipe@1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
     },
     "util-deprecate": {
diff --git a/package.json b/package.json
index 5cef5e9a..bc33e2a0 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
   ],
   "dependencies": {
     "basic-auth": "^2.0.0",
+    "body-parser":  "1.18.2",
     "bintrees": "1.0.1",
     "bunyan": "1.8.1",
     "cartodb-psql": "0.10.1",

From 2229d0ee5796da53355f6d618326f31fdbc1183f Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Fri, 13 Apr 2018 16:34:04 +0200
Subject: [PATCH 004/133] Fix jshint issues

---
 app/controllers/copy_controller.js | 12 ++----------
 app/middlewares/body-parser.js     |  2 +-
 app/utils/multer-pg-copy.js        | 12 +++++-------
 3 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 88d8fdd9..93a47619 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -1,16 +1,9 @@
 'use strict';
 
 var _ = require('underscore');
-var step = require('step');
-var assert = require('assert');
-var PSQL = require('cartodb-psql');
 var CachedQueryTables = require('../services/cached-query-tables');
-var queryMayWrite = require('../utils/query_may_write');
 
-var formats = require('../models/formats');
 
-var sanitize_filename = require('../utils/filename_sanitizer');
-var getContentDisposition = require('../utils/content_disposition');
 const userMiddleware = require('../middlewares/user');
 const errorMiddleware = require('../middlewares/error');
 const authorizationMiddleware = require('../middlewares/authorization');
@@ -20,7 +13,6 @@ const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 
-var ONE_YEAR_IN_SECONDS = 31536000; // 1 year time to live by default
 
 // We need NPM body-parser so we can use the multer and
 // still decode the urlencoded 'sql' parameter from
@@ -68,7 +60,7 @@ CopyController.prototype.route = function (app) {
 };
 
 // jshint maxcomplexity:21
-CopyController.prototype.handleCopyFrom = function (req, res, next) {
+CopyController.prototype.handleCopyFrom = function (req, res) {
 
     // Why doesn't this function do much of anything?
     // Because all the excitement is in the bodyParser and the upload
@@ -89,7 +81,7 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
         req.aborted = true; // TODO: there must be a builtin way to check this
     });
 
-    console.debug("CopyController.prototype.handleCopyFrom: sql = '%s'", req.body.sql)
+    console.debug("CopyController.prototype.handleCopyFrom: sql = '%s'", req.body.sql);
 
     res.send('got into handleCopyFrom');
         
diff --git a/app/middlewares/body-parser.js b/app/middlewares/body-parser.js
index 8bd5ff87..7bfb8bf9 100644
--- a/app/middlewares/body-parser.js
+++ b/app/middlewares/body-parser.js
@@ -11,7 +11,7 @@
  */
 
 var qs = require('qs');
-var multer = require('multer');
+//var multer = require('multer');
 
 /**
  * Extract the mime type from the given request's
diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
index f645a652..e6ace920 100644
--- a/app/utils/multer-pg-copy.js
+++ b/app/utils/multer-pg-copy.js
@@ -6,8 +6,6 @@ var _ = require('underscore');
 var fs = require('fs');
 var copyFrom = require('pg-copy-streams').from;
 
-var opts;
-
 function PgCopyCustomStorage (opts) {
   this.opts = opts || {};
 }
@@ -72,12 +70,12 @@ PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb)
       file.stream.pipe(outStream);
     });
   }
-}
+};
 
 PgCopyCustomStorage.prototype._removeFile = function _removeFile (req, file, cb) {
-  fs.unlink(file.path, cb)
-}
+    fs.unlink(file.path, cb);
+};
 
 module.exports = function (opts) {
-  return new PgCopyCustomStorage(opts)
-}
\ No newline at end of file
+    return new PgCopyCustomStorage(opts);
+};

From 72bce5732bc050cde4901108efd130497f4ee889 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Tue, 24 Apr 2018 11:26:15 +0200
Subject: [PATCH 005/133] WIP on /copyfrom

---
 app/controllers/copy_controller.js | 82 ++++++++++++++++++++++--------
 package.json                       |  4 +-
 2 files changed, 62 insertions(+), 24 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 93a47619..dab715c9 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -3,7 +3,6 @@
 var _ = require('underscore');
 var CachedQueryTables = require('../services/cached-query-tables');
 
-
 const userMiddleware = require('../middlewares/user');
 const errorMiddleware = require('../middlewares/error');
 const authorizationMiddleware = require('../middlewares/authorization');
@@ -13,6 +12,9 @@ const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 
+// Database requirements
+var PSQL = require('cartodb-psql');
+var fs = require('fs');
 
 // We need NPM body-parser so we can use the multer and
 // still decode the urlencoded 'sql' parameter from
@@ -26,10 +28,13 @@ var multer = require('multer');
 // do what we need, which is pipe the multer read stream
 // straight into the pg-copy write stream, so we use
 // a custom storage engine
-var multerpgcopy = require('../utils/multer-pg-copy');
-var upload = multer({ storage: multerpgcopy() });
+// var multerpgcopy = require('../utils/multer-pg-copy');
+// var upload = multer({ storage: multerpgcopy() });
 
-// var upload = multer({ dest: '/tmp/' });
+// Store the uploaded file in the tmp directory, with limits on the
+// size of acceptable uploads
+var uploadLimits = { fileSize: 1024*1024*1024, fields: 10, files: 1 };
+var upload = multer({ storage: multer.diskStorage({}), limits: uploadLimits });
 
 function CopyController(metadataBackend, userDatabaseService, tableCache, statsd_client, userLimitsService) {
     this.metadataBackend = metadataBackend;
@@ -60,31 +65,64 @@ CopyController.prototype.route = function (app) {
 };
 
 // jshint maxcomplexity:21
-CopyController.prototype.handleCopyFrom = function (req, res) {
+CopyController.prototype.handleCopyFrom = function (req, res, next) {
 
-    // Why doesn't this function do much of anything?
-    // Because all the excitement is in the bodyParser and the upload
-    // middlewards, the first of which should fill out the body.params.sql
-    // statement and the second of which should run that statement and 
-    // upload it into pgsql.
-    // All that's left here, is to read the number of records inserted
-    // and to return some information to the caller on what exactly 
-    // happened.
     // Test with:
     // curl --form file=@package.json --form sql="COPY this FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom
 
-    req.aborted = false;
-    req.on("close", function() {
-        if (req.formatter && _.isFunction(req.formatter.cancel)) {
-            req.formatter.cancel();
-        }
-        req.aborted = true; // TODO: there must be a builtin way to check this
-    });
 
-    console.debug("CopyController.prototype.handleCopyFrom: sql = '%s'", req.body.sql);
+    var sql = req.body.sql;
+    sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
 
-    res.send('got into handleCopyFrom');
+    // Ensure SQL parameter is not missing
+    if (!_.isString(sql)) {
+        throw new Error("Parameter 'sql' is missing");
+    }
+    
+    // Only accept SQL that starts with 'COPY'
+    if (!sql.toUpperCase().startsWith("COPY ")) {
+        throw new Error("SQL must start with COPY");
+    }
+    
+    // If SQL doesn't include "from stdin", add it
+    if (!sql.toUpperCase().endsWith(" FROM STDOUT")) {
+        sql += " FROM STDOUT";
+    }
+    
+    console.debug("CopyController.handleCopyFrom: sql = '%s'", sql);
+    
+    // The multer middleware should have filled 'req.file' in
+    // with the name, size, path, etc of the file
+    if (typeof req.file === 'undefined') {
+        throw new Error("Parameter 'file' is missing");
+    }
+    
+    try {        
+        // Open pgsql COPY pipe and stream file into it
+        const { user: username, userDbParams: dbopts, authDbParams, userLimits, authenticated } = res.locals;
+        var pg = new PSQL(authDbParams);
+        var copyFrom = require('pg-copy-streams').from;
+        var copyFromStream = copyFrom(sql);
+        // console.debug("XXX connect " + sql);
+        pg.dbConnect(pg.getConnectionConfig(), function(err, client, next) {
+            var stream = client.query(copyFromStream);
+            var fileStream = fs.createReadStream(req.file.path);
+            fileStream.on('error', next);
+            stream.on('error', next);
+            stream.on('end', next);
+            fileStream.pipe(stream)
+        });
+    } catch (err) {
+        console.debug("ERRROR!!!!")
+        next(err);
+    }
         
+    // Return some useful information about the process,
+    // pg-copy-streams fills in the rowCount after import is complete    
+    var result = "handleCopyFrom completed with row count = " + copyFromStream.rowCount;
+    console.debug("CopyController.handleCopyFrom: rowCount = '%s'", copyFromStream.rowCount);
+    
+    res.send(result + "\n");        
 };
 
 module.exports = CopyController;
diff --git a/package.json b/package.json
index bc33e2a0..2cd03dea 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,7 @@
   ],
   "dependencies": {
     "basic-auth": "^2.0.0",
-    "body-parser":  "1.18.2",
+    "body-parser":  "~1.18.2",
     "bintrees": "1.0.1",
     "bunyan": "1.8.1",
     "cartodb-psql": "0.10.1",
@@ -28,7 +28,7 @@
     "express": "~4.13.3",
     "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "~2.5.0",
-    "multer": "~1.2.0",
+    "multer": "~1.3.0",
     "node-statsd": "~0.0.7",
     "node-uuid": "^1.4.7",
     "oauth-client": "0.3.0",

From 4914100205f52abcfff25413941e936a2205ef99 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Tue, 24 Apr 2018 13:07:57 +0200
Subject: [PATCH 006/133] CopyFrom works, but still needs a decent return
 payload and a lot of work on returning useful information for error cases
 (post empty input, db errors returned more nicely? etc)

---
 app/controllers/copy_controller.js | 61 ++++++++++++++++++++----------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index dab715c9..d5e4caa6 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -46,9 +46,10 @@ function CopyController(metadataBackend, userDatabaseService, tableCache, statsd
 
 CopyController.prototype.route = function (app) {
     const { base_url } = global.settings;
+    
     const copyFromMiddlewares = endpointGroup => {
         return [
-            initializeProfilerMiddleware('query'),
+            initializeProfilerMiddleware('copyfrom'),
             userMiddleware(),
             rateLimitsMiddleware(this.userLimitsService, endpointGroup),
             authorizationMiddleware(this.metadataBackend),
@@ -60,16 +61,34 @@ CopyController.prototype.route = function (app) {
             errorMiddleware()
         ];
     };
+    
+    const copyToMiddlewares = endpointGroup => {
+        return [
+            initializeProfilerMiddleware('copyto'),
+            userMiddleware(),
+            rateLimitsMiddleware(this.userLimitsService, endpointGroup),
+            authorizationMiddleware(this.metadataBackend),
+            connectionParamsMiddleware(this.userDatabaseService),
+            timeoutLimitsMiddleware(this.metadataBackend),
+            this.handleCopyTo.bind(this),
+            errorMiddleware()
+        ];
+    };
 
     app.post(`${base_url}/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
+    app.get(`${base_url}/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
 };
 
+
+CopyController.prototype.handleCopyTo = function (req, res, next) {
+    res.send("/copyto Called\n");
+}
+
+
 // jshint maxcomplexity:21
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
 
-    // Test with:
-    // curl --form file=@package.json --form sql="COPY this FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom
-
+    // curl --form file=@copyfrom.txt --form sql="COPY foo FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom
 
     var sql = req.body.sql;
     sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
@@ -85,44 +104,44 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
     }
     
     // If SQL doesn't include "from stdin", add it
-    if (!sql.toUpperCase().endsWith(" FROM STDOUT")) {
-        sql += " FROM STDOUT";
+    if (!sql.toUpperCase().endsWith(" FROM STDIN")) {
+        sql += " FROM STDIN";
     }
     
-    console.debug("CopyController.handleCopyFrom: sql = '%s'", sql);
+    // console.debug("CopyController.handleCopyFrom: sql = '%s'", sql);
     
     // The multer middleware should have filled 'req.file' in
     // with the name, size, path, etc of the file
     if (typeof req.file === 'undefined') {
         throw new Error("Parameter 'file' is missing");
     }
-    
+
+    var copyFrom = require('pg-copy-streams').from;
+    var copyFromStream = copyFrom(sql);
+
+    var returnResult = function() {
+        // Return some useful information about the process,
+        // pg-copy-streams fills in the rowCount after import is complete
+        var result = "handleCopyFrom completed with row count = " + copyFromStream.rowCount;
+        res.send(result + "\n");
+    }
+        
     try {        
-        // Open pgsql COPY pipe and stream file into it
+        // Open pgsql COPY pipe and stream in tmp file
         const { user: username, userDbParams: dbopts, authDbParams, userLimits, authenticated } = res.locals;
         var pg = new PSQL(authDbParams);
-        var copyFrom = require('pg-copy-streams').from;
-        var copyFromStream = copyFrom(sql);
-        // console.debug("XXX connect " + sql);
-        pg.dbConnect(pg.getConnectionConfig(), function(err, client, next) {
+        pg.connect(function(err, client, cb) {
             var stream = client.query(copyFromStream);
             var fileStream = fs.createReadStream(req.file.path);
             fileStream.on('error', next);
             stream.on('error', next);
-            stream.on('end', next);
+            stream.on('end', returnResult);
             fileStream.pipe(stream)
         });
     } catch (err) {
-        console.debug("ERRROR!!!!")
         next(err);
     }
-        
-    // Return some useful information about the process,
-    // pg-copy-streams fills in the rowCount after import is complete    
-    var result = "handleCopyFrom completed with row count = " + copyFromStream.rowCount;
-    console.debug("CopyController.handleCopyFrom: rowCount = '%s'", copyFromStream.rowCount);
     
-    res.send(result + "\n");        
 };
 
 module.exports = CopyController;

From b59bda5780d664a444c38202209896c3c318c348 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Tue, 24 Apr 2018 15:55:20 +0200
Subject: [PATCH 007/133] Get working copyto implementation in place. Many test
 cases needed still and thoughts on how to communicate errors back that are
 maybe nicer than stack messages?

---
 app/controllers/copy_controller.js | 48 +++++++++++++++++++++++++-----
 1 file changed, 41 insertions(+), 7 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index d5e4caa6..e748386d 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -15,6 +15,8 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 // Database requirements
 var PSQL = require('cartodb-psql');
 var fs = require('fs');
+var copyTo = require('pg-copy-streams').to;
+var copyFrom = require('pg-copy-streams').from;
 
 // We need NPM body-parser so we can use the multer and
 // still decode the urlencoded 'sql' parameter from
@@ -81,7 +83,45 @@ CopyController.prototype.route = function (app) {
 
 
 CopyController.prototype.handleCopyTo = function (req, res, next) {
-    res.send("/copyto Called\n");
+    
+    // curl "http://cdb.localhost.lan:8080/api/v2/copyto?sql=copy+foo+to+stdout&filename=output.dmp"
+        
+    var sql = req.query.sql;
+    var filename = req.query.filename;
+    sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
+    
+    // Ensure SQL parameter is not missing
+    if (!_.isString(sql)) {
+        throw new Error("Parameter 'sql' is missing");
+    }
+    
+    // console.debug("CopyController.prototype.handleCopyTo: sql = '%s'", sql);
+    
+    // Only accept SQL that starts with 'COPY'
+    if (!sql.toUpperCase().startsWith("COPY ")) {
+        throw new Error("SQL must start with COPY");
+    }
+    
+    try {        
+        // Open pgsql COPY pipe and stream out to HTTP response
+        const { user: username, userDbParams: dbopts, authDbParams, userLimits, authenticated } = res.locals;
+        var pg = new PSQL(authDbParams);
+        pg.connect(function(err, client, cb) {
+            var copyToStream = copyTo(sql);
+            var pgstream = client.query(copyToStream);
+            res.on('error', next);
+            pgstream.on('error', next);
+            pgstream.on('end', cb);
+            if (_.isString(filename)) {
+                var contentDisposition = "attachment; filename*=UTF-8''" + encodeURIComponent(filename);
+                res.setHeader("Content-Disposition", contentDisposition);
+            }
+            res.setHeader("Content-Type", "application/octet-stream");
+            pgstream.pipe(res)
+        });
+    } catch (err) {
+        next(err);
+    }    
 }
 
 
@@ -103,11 +143,6 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
         throw new Error("SQL must start with COPY");
     }
     
-    // If SQL doesn't include "from stdin", add it
-    if (!sql.toUpperCase().endsWith(" FROM STDIN")) {
-        sql += " FROM STDIN";
-    }
-    
     // console.debug("CopyController.handleCopyFrom: sql = '%s'", sql);
     
     // The multer middleware should have filled 'req.file' in
@@ -116,7 +151,6 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
         throw new Error("Parameter 'file' is missing");
     }
 
-    var copyFrom = require('pg-copy-streams').from;
     var copyFromStream = copyFrom(sql);
 
     var returnResult = function() {

From a98781d335fb2155bed0177fb2fabb9deead466a Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Wed, 25 Apr 2018 18:37:04 +0200
Subject: [PATCH 008/133] Working code for /copyfrom (streaming) and /copyto
 (streaming)

---
 app/controllers/copy_controller.js |  82 ++++++++--------------
 app/utils/multer-pg-copy.js        | 107 +++++++++++++++--------------
 2 files changed, 83 insertions(+), 106 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index e748386d..39603052 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -16,7 +16,6 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 var PSQL = require('cartodb-psql');
 var fs = require('fs');
 var copyTo = require('pg-copy-streams').to;
-var copyFrom = require('pg-copy-streams').from;
 
 // We need NPM body-parser so we can use the multer and
 // still decode the urlencoded 'sql' parameter from
@@ -30,13 +29,13 @@ var multer = require('multer');
 // do what we need, which is pipe the multer read stream
 // straight into the pg-copy write stream, so we use
 // a custom storage engine
-// var multerpgcopy = require('../utils/multer-pg-copy');
-// var upload = multer({ storage: multerpgcopy() });
+var multerpgcopy = require('../utils/multer-pg-copy');
+var upload = multer({ storage: multerpgcopy() });
 
 // Store the uploaded file in the tmp directory, with limits on the
 // size of acceptable uploads
-var uploadLimits = { fileSize: 1024*1024*1024, fields: 10, files: 1 };
-var upload = multer({ storage: multer.diskStorage({}), limits: uploadLimits });
+// var uploadLimits = { fileSize: 1024*1024*1024, fields: 10, files: 1 };
+// var upload = multer({ storage: multer.diskStorage({}), limits: uploadLimits });
 
 function CopyController(metadataBackend, userDatabaseService, tableCache, statsd_client, userLimitsService) {
     this.metadataBackend = metadataBackend;
@@ -49,6 +48,8 @@ function CopyController(metadataBackend, userDatabaseService, tableCache, statsd
 CopyController.prototype.route = function (app) {
     const { base_url } = global.settings;
     
+    console.debug("CopyController.prototype.route");
+    
     const copyFromMiddlewares = endpointGroup => {
         return [
             initializeProfilerMiddleware('copyfrom'),
@@ -57,6 +58,7 @@ CopyController.prototype.route = function (app) {
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
+            this.copyDbParamsToReq.bind(this),
             bodyParser.urlencoded({ extended: true }),
             upload.single('file'),
             this.handleCopyFrom.bind(this),
@@ -81,9 +83,14 @@ CopyController.prototype.route = function (app) {
     app.get(`${base_url}/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
 };
 
+CopyController.prototype.copyDbParamsToReq = function (req, res, next) {
+    const { user: username, userDbParams: dbopts, authDbParams, userLimits, authenticated } = res.locals;
+    req.authDbParams = authDbParams;
+    next();    
+}
 
 CopyController.prototype.handleCopyTo = function (req, res, next) {
-    
+        
     // curl "http://cdb.localhost.lan:8080/api/v2/copyto?sql=copy+foo+to+stdout&filename=output.dmp"
         
     var sql = req.query.sql;
@@ -94,9 +101,7 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
     if (!_.isString(sql)) {
         throw new Error("Parameter 'sql' is missing");
     }
-    
-    // console.debug("CopyController.prototype.handleCopyTo: sql = '%s'", sql);
-    
+        
     // Only accept SQL that starts with 'COPY'
     if (!sql.toUpperCase().startsWith("COPY ")) {
         throw new Error("SQL must start with COPY");
@@ -121,61 +126,30 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
         });
     } catch (err) {
         next(err);
-    }    
+    }
+    
 }
 
 
 // jshint maxcomplexity:21
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
 
+    // All the action happens in multer, which reads the incoming
+    // file into a stream, and then hands it to the custom storage
+    // engine defined in multer-pg-copy.js.
+    // The storage engine writes the rowCount into req when it's 
+    // finished. Hopefully any errors just propogate up.
+    
     // curl --form file=@copyfrom.txt --form sql="COPY foo FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom
 
-    var sql = req.body.sql;
-    sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
-
-    // Ensure SQL parameter is not missing
-    if (!_.isString(sql)) {
-        throw new Error("Parameter 'sql' is missing");
+    if (typeof req.rowCount === "undefined")
+    {
+        throw new Error("no rows copied");
     }
     
-    // Only accept SQL that starts with 'COPY'
-    if (!sql.toUpperCase().startsWith("COPY ")) {
-        throw new Error("SQL must start with COPY");
-    }
-    
-    // console.debug("CopyController.handleCopyFrom: sql = '%s'", sql);
-    
-    // The multer middleware should have filled 'req.file' in
-    // with the name, size, path, etc of the file
-    if (typeof req.file === 'undefined') {
-        throw new Error("Parameter 'file' is missing");
-    }
-
-    var copyFromStream = copyFrom(sql);
-
-    var returnResult = function() {
-        // Return some useful information about the process,
-        // pg-copy-streams fills in the rowCount after import is complete
-        var result = "handleCopyFrom completed with row count = " + copyFromStream.rowCount;
-        res.send(result + "\n");
-    }
-        
-    try {        
-        // Open pgsql COPY pipe and stream in tmp file
-        const { user: username, userDbParams: dbopts, authDbParams, userLimits, authenticated } = res.locals;
-        var pg = new PSQL(authDbParams);
-        pg.connect(function(err, client, cb) {
-            var stream = client.query(copyFromStream);
-            var fileStream = fs.createReadStream(req.file.path);
-            fileStream.on('error', next);
-            stream.on('error', next);
-            stream.on('end', returnResult);
-            fileStream.pipe(stream)
-        });
-    } catch (err) {
-        next(err);
-    }
+    var result = "handleCopyFrom completed with row count = " + req.rowCount;
+    res.send(result + "\n");
     
 };
 
-module.exports = CopyController;
+module.exports = CopyController;
\ No newline at end of file
diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
index e6ace920..e4d1bb21 100644
--- a/app/utils/multer-pg-copy.js
+++ b/app/utils/multer-pg-copy.js
@@ -5,71 +5,74 @@
 var _ = require('underscore');
 var fs = require('fs');
 var copyFrom = require('pg-copy-streams').from;
+var PSQL = require('cartodb-psql');
 
 function PgCopyCustomStorage (opts) {
-  this.opts = opts || {};
+    this.opts = opts || {};
 }
 
 PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb) {
 
-  // Skip the pg-copy for now, just write to /tmp/
-  // so we can see what parameters are making it into 
-  // this storage handler
-  var debug_customstorage = true; 
-
-  // Hopefully the body-parser has extracted the 'sql' parameter
-  // Otherwise, this will be a short trip, as we won't be able
-  // to run the pg-copy-streams
-  var sql = req.body.sql;
-  sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
-
-  console.debug("PgCopyCustomStorage.prototype._handleFile");
-  console.debug("PgCopyCustomStorage.prototype._handleFile: sql = '%s'", sql);
+    // Hopefully the body-parser has extracted the 'sql' parameter
+    // or the user has provided it on the URL line.
+    // Otherwise, this will be a short trip, as we won't be able
+    // to the pg-copy-streams SQL command
+    var b_sql = req.body.sql;
+    b_sql = (b_sql === "" || _.isUndefined(b_sql)) ? null : b_sql;
+    var q_sql = req.query.sql;
+    q_sql = (q_sql === "" || _.isUndefined(q_sql)) ? null : q_sql;
+    var sql = b_sql || q_sql;
   
-  if (debug_customstorage) {
-    var outStream = fs.createWriteStream('/tmp/sqlApiUploadExample');
-    file.stream.pipe(outStream);
-    outStream.on('error', cb);
-    outStream.on('finish', function () {
-      cb(null, {
-        path: file.path,
-        size: outStream.bytesWritten
-      });
-    });
+    // Ensure SQL parameter is not missing
+    if (!_.isString(sql)) {
+        cb(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
+    }
     
-  } else {
-    // TODO, handle this nicely
-    if(!_.isString(sql)) {
-      throw new Error("sql is not set");
+    // Only accept SQL that starts with 'COPY'
+    if (!sql.toUpperCase().startsWith("COPY ")) {
+        cb(new Error("SQL must start with COPY"));
+    }    
+
+    // We expect the an earlier middleware to have 
+    // set this by the time we are called via multer,
+    // so this should never happen
+    if (!req.authDbParams) {
+        cb(new Error("req.authDbParams is not set"));
     }
 
-    // We expect the pg-connect middleware to have 
-    // set this by the time we are called via multer
-    if (!req.authDbConnection) {
-      throw new Error("req.authDbConnection is not set");
+    var copyFromStream = copyFrom(sql);
+
+    var returnResult = function() {
+        // Fill in the rowCount on the request (because we don't have)
+        // access to the response here, so that the final handler 
+        // can return a response
+        req.rowCount = copyFromStream.rowCount;
+        
     }
-    var sessionPg = req.authDbConnection;
+        
+    try {
+        // Connect and run the COPY
+        var pg = new PSQL(req.authDbParams);
+        
+        pg.connect(function(err, client, done) {
+            if (err) {
+                return done(err);
+            }
+            var pgstream = client.query(copyFromStream);
+            file.stream.on('error', cb);
+            pgstream.on('error', cb);
+            pgstream.on('end', function () {
+                req.rowCount = copyFromStream.rowCount;
+                cb(null, {rowCount: copyFromStream.rowCount});
+            });
+            file.stream.pipe(pgstream);
+        });
 
-    sessionPg.connect(function(err, client, done) {
-      if (err) {
-        return cb(err);
-      }
+    } catch (err) {
+        cb(err);
+    }
+    return 
 
-      console.debug("XXX pg.connect");
-
-      // This is the magic part, see 
-      // https://github.com/brianc/node-pg-copy-streams
-      var outStream = client.query(copyFrom(sql), function(err, result) {
-        done(err);
-        return cb(err, result);
-      });
-
-      file.stream.on('error', cb);
-      outStream.on('error', cb);
-      outStream.on('end', cb);
-      file.stream.pipe(outStream);
-    });
-  }
 };
 
 PgCopyCustomStorage.prototype._removeFile = function _removeFile (req, file, cb) {

From ea66076255eb6887a181971bd3994d1fa632a80e Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 26 Apr 2018 09:40:13 +0200
Subject: [PATCH 009/133] Remove 'console.debug', try and get travis tests
 clean

---
 app/controllers/copy_controller.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 39603052..6dba094d 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -47,9 +47,7 @@ function CopyController(metadataBackend, userDatabaseService, tableCache, statsd
 
 CopyController.prototype.route = function (app) {
     const { base_url } = global.settings;
-    
-    console.debug("CopyController.prototype.route");
-    
+        
     const copyFromMiddlewares = endpointGroup => {
         return [
             initializeProfilerMiddleware('copyfrom'),

From cc5fa5c6ce8bb7ccce05ecf39b55466c6c8196ce Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 26 Apr 2018 10:04:38 +0200
Subject: [PATCH 010/133] Changes suggested by jshint

---
 app/controllers/copy_controller.js | 21 +++++++++++----------
 app/utils/multer-pg-copy.js        | 10 +---------
 2 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 6dba094d..c2f2de43 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -14,7 +14,6 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 
 // Database requirements
 var PSQL = require('cartodb-psql');
-var fs = require('fs');
 var copyTo = require('pg-copy-streams').to;
 
 // We need NPM body-parser so we can use the multer and
@@ -82,10 +81,10 @@ CopyController.prototype.route = function (app) {
 };
 
 CopyController.prototype.copyDbParamsToReq = function (req, res, next) {
-    const { user: username, userDbParams: dbopts, authDbParams, userLimits, authenticated } = res.locals;
-    req.authDbParams = authDbParams;
+
+    req.authDbParams = res.locals.authDbParams;
     next();    
-}
+};
 
 CopyController.prototype.handleCopyTo = function (req, res, next) {
         
@@ -107,7 +106,7 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
     
     try {        
         // Open pgsql COPY pipe and stream out to HTTP response
-        const { user: username, userDbParams: dbopts, authDbParams, userLimits, authenticated } = res.locals;
+        const authDbParams = res.locals.authDbParams;
         var pg = new PSQL(authDbParams);
         pg.connect(function(err, client, cb) {
             var copyToStream = copyTo(sql);
@@ -120,13 +119,13 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
                 res.setHeader("Content-Disposition", contentDisposition);
             }
             res.setHeader("Content-Type", "application/octet-stream");
-            pgstream.pipe(res)
+            pgstream.pipe(res);
         });
     } catch (err) {
         next(err);
     }
     
-}
+};
 
 
 // jshint maxcomplexity:21
@@ -138,14 +137,16 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
     // The storage engine writes the rowCount into req when it's 
     // finished. Hopefully any errors just propogate up.
     
-    // curl --form file=@copyfrom.txt --form sql="COPY foo FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom
+    // curl --form sql="COPY foo FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom --form file=@copyfrom.txt 
 
-    if (typeof req.rowCount === "undefined")
+
+    if (typeof req.file === "undefined" || typeof req.file.rowCount === "undefined")
     {
         throw new Error("no rows copied");
     }
+    var rowCount = req.file.rowCount;
     
-    var result = "handleCopyFrom completed with row count = " + req.rowCount;
+    var result = result + "handleCopyFrom completed with row count = " + rowCount;
     res.send(result + "\n");
     
 };
diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
index e4d1bb21..f8b5c808 100644
--- a/app/utils/multer-pg-copy.js
+++ b/app/utils/multer-pg-copy.js
@@ -42,14 +42,6 @@ PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb)
 
     var copyFromStream = copyFrom(sql);
 
-    var returnResult = function() {
-        // Fill in the rowCount on the request (because we don't have)
-        // access to the response here, so that the final handler 
-        // can return a response
-        req.rowCount = copyFromStream.rowCount;
-        
-    }
-        
     try {
         // Connect and run the COPY
         var pg = new PSQL(req.authDbParams);
@@ -71,7 +63,7 @@ PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb)
     } catch (err) {
         cb(err);
     }
-    return 
+    return;
 
 };
 

From f13028b497cc42c351d2ebf549bc2aa3b0bf7917 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 26 Apr 2018 10:20:21 +0200
Subject: [PATCH 011/133] More jshint changes

---
 app/controllers/copy_controller.js | 15 ++++++++-------
 app/utils/multer-pg-copy.js        | 24 +++++++++++++++++-------
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index c2f2de43..463b9518 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -129,7 +129,7 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
 
 
 // jshint maxcomplexity:21
-CopyController.prototype.handleCopyFrom = function (req, res, next) {
+CopyController.prototype.handleCopyFrom = function (req, res) {
 
     // All the action happens in multer, which reads the incoming
     // file into a stream, and then hands it to the custom storage
@@ -140,14 +140,15 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
     // curl --form sql="COPY foo FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom --form file=@copyfrom.txt 
 
 
-    if (typeof req.file === "undefined" || typeof req.file.rowCount === "undefined")
-    {
+    if (typeof req.file === "undefined") {
         throw new Error("no rows copied");
     }
-    var rowCount = req.file.rowCount;
-    
-    var result = result + "handleCopyFrom completed with row count = " + rowCount;
-    res.send(result + "\n");
+    var msg = {time: req.file.time, total_rows: req.file.total_rows};
+    if (req.query && req.query.callback) {
+        res.jsonp(msg);
+    } else {
+        res.json(msg);
+    }
     
 };
 
diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
index f8b5c808..69ae1680 100644
--- a/app/utils/multer-pg-copy.js
+++ b/app/utils/multer-pg-copy.js
@@ -11,17 +11,23 @@ function PgCopyCustomStorage (opts) {
     this.opts = opts || {};
 }
 
+// Pick the URL or the body SQL query, prefering the body query
+// if both are filled in (???)
+function getSqlParam(req) {
+    var b_sql = req.body.sql;
+    b_sql = (b_sql === "" || _.isUndefined(b_sql)) ? null : b_sql;
+    var q_sql = req.query.sql;
+    q_sql = (q_sql === "" || _.isUndefined(q_sql)) ? null : q_sql;
+    return b_sql || q_sql;
+}
+
 PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb) {
 
     // Hopefully the body-parser has extracted the 'sql' parameter
     // or the user has provided it on the URL line.
     // Otherwise, this will be a short trip, as we won't be able
     // to the pg-copy-streams SQL command
-    var b_sql = req.body.sql;
-    b_sql = (b_sql === "" || _.isUndefined(b_sql)) ? null : b_sql;
-    var q_sql = req.query.sql;
-    q_sql = (q_sql === "" || _.isUndefined(q_sql)) ? null : q_sql;
-    var sql = b_sql || q_sql;
+    var sql = getSqlParam(req);
   
     // Ensure SQL parameter is not missing
     if (!_.isString(sql)) {
@@ -45,6 +51,7 @@ PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb)
     try {
         // Connect and run the COPY
         var pg = new PSQL(req.authDbParams);
+        var start_time = Date.now();
         
         pg.connect(function(err, client, done) {
             if (err) {
@@ -54,8 +61,11 @@ PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb)
             file.stream.on('error', cb);
             pgstream.on('error', cb);
             pgstream.on('end', function () {
-                req.rowCount = copyFromStream.rowCount;
-                cb(null, {rowCount: copyFromStream.rowCount});
+                var end_time = Date.now();
+                cb(null, {
+                    total_rows: copyFromStream.rowCount,
+                    time: (end_time - start_time)/1000
+                });
             });
             file.stream.pipe(pgstream);
         });

From c78cac9bd6dcc3e12cf3bf9b3a1277c8825dd790 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 26 Apr 2018 10:45:52 +0200
Subject: [PATCH 012/133] Use the correct database credentials, so we obey
 security

---
 app/controllers/copy_controller.js | 5 ++---
 app/utils/multer-pg-copy.js        | 6 +++---
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 463b9518..c191dd23 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -82,7 +82,7 @@ CopyController.prototype.route = function (app) {
 
 CopyController.prototype.copyDbParamsToReq = function (req, res, next) {
 
-    req.authDbParams = res.locals.authDbParams;
+    req.userDbParams = res.locals.userDbParams;
     next();    
 };
 
@@ -106,8 +106,7 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
     
     try {        
         // Open pgsql COPY pipe and stream out to HTTP response
-        const authDbParams = res.locals.authDbParams;
-        var pg = new PSQL(authDbParams);
+        var pg = new PSQL(res.locals.userDbParams);
         pg.connect(function(err, client, cb) {
             var copyToStream = copyTo(sql);
             var pgstream = client.query(copyToStream);
diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
index 69ae1680..ef931119 100644
--- a/app/utils/multer-pg-copy.js
+++ b/app/utils/multer-pg-copy.js
@@ -42,15 +42,15 @@ PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb)
     // We expect the an earlier middleware to have 
     // set this by the time we are called via multer,
     // so this should never happen
-    if (!req.authDbParams) {
-        cb(new Error("req.authDbParams is not set"));
+    if (!req.userDbParams) {
+        cb(new Error("req.userDbParams is not set"));
     }
 
     var copyFromStream = copyFrom(sql);
 
     try {
         // Connect and run the COPY
-        var pg = new PSQL(req.authDbParams);
+        var pg = new PSQL(req.userDbParams);
         var start_time = Date.now();
         
         pg.connect(function(err, client, done) {

From fe52af71ac8b32127a8f346619183da543fd1fba Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Fri, 27 Apr 2018 11:55:06 +0200
Subject: [PATCH 013/133] Add documentation SQL API copyfrom/copyto end points,
 and make small modifications arising from that task

---
 app/controllers/copy_controller.js |   6 +-
 doc/API.md                         |   1 +
 doc/copy_queries.md                | 153 +++++++++++++++++++++++++++++
 3 files changed, 159 insertions(+), 1 deletion(-)
 create mode 100644 doc/copy_queries.md

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index c191dd23..573ba5c2 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -114,7 +114,11 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
             pgstream.on('error', next);
             pgstream.on('end', cb);
             if (_.isString(filename)) {
-                var contentDisposition = "attachment; filename*=UTF-8''" + encodeURIComponent(filename);
+                var contentDisposition = "attachment; filename=" + encodeURIComponent(filename);
+                res.setHeader("Content-Disposition", contentDisposition);
+            } else {
+                filename = 'carto-sql-copyto.dmp';
+                var contentDisposition = "attachment; filename=" + encodeURIComponent(filename);
                 res.setHeader("Content-Disposition", contentDisposition);
             }
             res.setHeader("Content-Type", "application/octet-stream");
diff --git a/doc/API.md b/doc/API.md
index aba951bc..f367730e 100644
--- a/doc/API.md
+++ b/doc/API.md
@@ -17,6 +17,7 @@ Remember that in order to access, read or modify data in private tables, you wil
 * [Making calls to the SQL API](making_calls.md)
 * [Creating Tables with the SQL API](creating_tables.md)
 * [Batch Queries](batch_queries.md)
+* [Copy Queries](copy_queries.md)
 * [Handling Geospatial Data](handling_geospatial_data.md)
 * [Query Optimizations](query_optimizations.md)
 * [API Version Vumber](version.md)
diff --git a/doc/copy_queries.md b/doc/copy_queries.md
new file mode 100644
index 00000000..befd28ba
--- /dev/null
+++ b/doc/copy_queries.md
@@ -0,0 +1,153 @@
+# Copy Queries
+
+Copy queries allow you to use the [PostgreSQL copy command](https://www.postgresql.org/docs/10/static/sql-copy.html) for efficient streaming of data to and from CARTO.
+
+The support for copy is split across two API end points:
+
+* `http://{username}.carto.com/api/v2/copyfrom` for uploading data to CARTO
+* `http://{username}.carto.com/api/v2/copyto` for exporting data out of CARTO
+
+## Copy From
+
+The PostgreSQL `COPY` command is extremely fast, but requires very precise inputs:
+
+* A `COPY` command that describes the table and columns of the upload file, and the format of the file.
+* An upload file that exactly matches the `COPY` command.
+
+If the `COPY` command and the target table do not match, the upload will fail.
+
+In addition, for a table to be readable by CARTO, it must have a minimum of three columns with specific data types:
+
+* `cartodb_id`, a `serial` primary key 
+* `the_geom`, a geometry in the ESPG:4326 (aka long/lat) projection
+* `the_geom_webmercator`, a geometry in the ESPG:3857 (aka web mercator) projection
+
+Creating a new CARTO table with all the right triggers and columns can is tricky, so here is an example:
+
+    -- create the table using the *required* columns and a 
+    -- couple more
+    CREATE TABLE upload_example (
+        the_geom geometry,
+        name text,
+        age integer
+    );
+
+    -- adds the 'cartodb_id' and 'the_geom_webmercator'
+    -- adds the required triggers and indexes
+    SELECT CDB_CartodbfyTable('upload_example');
+    
+Now you are read to upload your file. Suppose you have a CSV file like this:
+
+    the_geom,name,age
+    SRID=4326;POINT(-126 54),North West,89
+    SRID=4326;POINT(-96 34),South East,99
+    SRID=4326;POINT(-6 -25),Souther Easter,124
+
+The `COPY` command to upload this file needs to specify the file format (CSV), the fact that there is a header line before the actual data begins, and enumerate the columns that are in the file so they can be matched to the table columns.
+
+    COPY upload_example (the_geom, name, age) 
+    FROM STDIN WITH (FORMAT csv, HEADER true)
+
+The `FROM STDIN` option tells the database that the input will come from a data stream, and the SQL API will read our uploaded file and use it to feed the stream.
+
+To actually run upload, you will need a tool or script that can generate a `multipart/form-data` POST, so here are a few examples in different languages.
+
+### CURL Example
+
+The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from the commandline, and supports multi-part file upload, so it can feed the `copyfrom` end point.
+
+Assuming that you have already created the table, and that the CSV file is named "upload_example.csv":
+
+    curl \
+      --form sql="COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)" \
+      --form file=@upload_example.csv \
+      http://{username}.carto.com/api/v2/copyfrom?api_key={api_key}
+
+**Important:** When supplying the "sql" parameter as a form field, it must come **before** the "file" parameter, or the upload will fail. Alternatively, you can supply the "sql" parameter on the URL line.
+
+### Python Example
+
+The [Requests](http://docs.python-requests.org/en/master/user/quickstart/) library for HTTP makes doing a file upload relatively terse. 
+
+    import requests
+
+    api_key = {api_key}
+    username = {api_key}
+    upload_file = 'upload_example.csv'
+    sql = "COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)"
+
+    url = "http://%s.carto.com/api/v2/copyfrom" % username    
+    with open(upload_file, 'rb') as f:
+        r = requests.post(url, params={'api_key':api_key, 'sql':sql}, files={'file': f})
+        if r.status_code != 200:
+            print r.text
+        else:
+            status = r.json()
+            print "Success: %s rows imported" % status['total_rows']
+
+A slightly more sophisticated script could read the headers from the CSV and compose the `COPY` command on the fly.
+
+### Response Format
+
+A successful upload will return with status code 200, and a small JSON with information about the upload.
+
+    {"time":0.046,"total_rows":10}
+
+A failed upload will return with status code 400 and a larger JSON with the PostgreSQL error string, and a stack trace from the SQL API.
+
+    {"error":["Unexpected field"],
+     "stack":"Error: Unexpected field
+        at makeError (/repos/CartoDB-SQL-API/node_modules/multer/lib/make-error.js:12:13)
+        at wrappedFileFilter (/repos/CartoDB-SQL-API/node_modules/multer/index.js:39:19)
+        ...
+        at emitMany (events.js:127:13)
+        at SBMH.emit (events.js:201:7)"}
+    
+## Copy To
+
+Using the `copyto` end point to extract data bypasses the usual JSON formatting applied by the SQL API, so it can dump more data, faster. However, it has the restriction that it will only output in a handful of formats:
+
+* PgSQL [text format](https://www.postgresql.org/docs/10/static/sql-copy.html#id-1.9.3.52.9.2),
+* [CSV](https://www.postgresql.org/docs/10/static/sql-copy.html#id-1.9.3.52.9.3), and
+* PgSQL [binary format](https://www.postgresql.org/docs/10/static/sql-copy.html#id-1.9.3.52.9.4).
+
+"Copy to" is a simple HTTP GET end point, so any tool or language can be easily used to download data, supplying the following parameters in the URL:
+
+* `sql`, the "COPY" command to extract the data.
+* `filename`, the filename to put in the "Content-disposition" HTTP header. Useful for tools that automatically save the download to a file name.
+* `api_key`, your API key for reading non-public tables.
+
+
+### CURL Example
+
+    curl \
+        --output upload_example_dl.csv \
+        "http://{username}.carto.com/api/v2/copyfrom?sql=COPY+upload_example+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
+
+### Python Example
+
+    import requests
+    import re
+
+    api_key = {api_key}
+    username = {api_key}
+    download_file = 'upload_example_dl.csv'
+    sql = "COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)"
+
+    # request the download, specifying desired file name
+    url = "http://%s.carto.com/api/v2/copyto" % username    
+    r = requests.get(url, params={'api_key':api_key, 'sql':sql, 'filename':download_file})
+    r.raise_for_status()
+
+    # read save file name from response headers
+    d = r.headers['content-disposition']
+    savefilename = re.findall("filename=(.+)", d)
+
+    if len(savefilename) > 0:
+        with open(savefilename[0], 'wb') as handle:
+            for block in r.iter_content(1024):
+                handle.write(block)        
+        print "Downloaded to: %s" % savefilename
+    else:
+        print "Error: could not find read file name from headers"
+

From f4dd157e4fd7529086898394856dfd1433076d3c Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Fri, 27 Apr 2018 12:05:36 +0200
Subject: [PATCH 014/133] Fix typo in example

---
 doc/copy_queries.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index befd28ba..35f17c56 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -122,7 +122,7 @@ Using the `copyto` end point to extract data bypasses the usual JSON formatting
 
     curl \
         --output upload_example_dl.csv \
-        "http://{username}.carto.com/api/v2/copyfrom?sql=COPY+upload_example+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
+        "http://{username}.carto.com/api/v2/copyto?sql=COPY+upload_example+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
 
 ### Python Example
 

From fcea39adba95b92f0b67ba98c26e3466dc62befd Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Fri, 27 Apr 2018 12:17:38 +0200
Subject: [PATCH 015/133] Slight rewrite and fixes.

---
 doc/copy_queries.md | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 35f17c56..4fc15f99 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -16,11 +16,21 @@ The PostgreSQL `COPY` command is extremely fast, but requires very precise input
 
 If the `COPY` command and the target table do not match, the upload will fail.
 
-In addition, for a table to be readable by CARTO, it must have a minimum of three columns with specific data types:
+"Copy from" copies data "from" your file, "to" CARTO. "Copy from" uses [multipart/form-data](https://stackoverflow.com/questions/8659808/how-does-http-file-upload-work) to stream an upload file to the server. This avoids limitations around file size and any need for temporary storage: the data travels from your file straight into the database.
 
-* `cartodb_id`, a `serial` primary key 
-* `the_geom`, a geometry in the ESPG:4326 (aka long/lat) projection
-* `the_geom_webmercator`, a geometry in the ESPG:3857 (aka web mercator) projection
+* `api_key` provided in the request URL parameters.
+* `sql` provided either in the request URL parameters or in a multipart form variable.
+* `file` provided as a multipart form variable; this is the actual file content, not a filename.
+
+Composing a multipart form data upload is moderately complicated, so almost all developers will use a tool or scripting language to upload data to CARTO via "copy from". 
+
+### Example
+
+For a table to be readable by CARTO, it must have a minimum of three columns with specific data types:
+
+* `cartodb_id`, a `serial` primary key
+* `the_geom`, a `geometry` in the ESPG:4326 projection (aka long/lat)
+* `the_geom_webmercator`, a `geometry` in the ESPG:3857 projection (aka web mercator)
 
 Creating a new CARTO table with all the right triggers and columns can is tricky, so here is an example:
 
@@ -36,14 +46,14 @@ Creating a new CARTO table with all the right triggers and columns can is tricky
     -- adds the required triggers and indexes
     SELECT CDB_CartodbfyTable('upload_example');
     
-Now you are read to upload your file. Suppose you have a CSV file like this:
+Now you are ready to upload your file. Suppose you have a CSV file like this:
 
     the_geom,name,age
     SRID=4326;POINT(-126 54),North West,89
     SRID=4326;POINT(-96 34),South East,99
     SRID=4326;POINT(-6 -25),Souther Easter,124
 
-The `COPY` command to upload this file needs to specify the file format (CSV), the fact that there is a header line before the actual data begins, and enumerate the columns that are in the file so they can be matched to the table columns.
+The `COPY` command to upload this file needs to specify the file format (CSV), the fact that there is a header line before the actual data begins, and to enumerate the columns that are in the file so they can be matched to the table columns.
 
     COPY upload_example (the_geom, name, age) 
     FROM STDIN WITH (FORMAT csv, HEADER true)
@@ -54,7 +64,7 @@ To actually run upload, you will need a tool or script that can generate a `mult
 
 ### CURL Example
 
-The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from the commandline, and supports multi-part file upload, so it can feed the `copyfrom` end point.
+The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from the command-line, and supports multi-part file upload, so it can feed the `copyfrom` end point.
 
 Assuming that you have already created the table, and that the CSV file is named "upload_example.csv":
 
@@ -105,6 +115,8 @@ A failed upload will return with status code 400 and a larger JSON with the Post
     
 ## Copy To
 
+"Copy to" copies data "to" your desired output file, "from" CARTO.
+
 Using the `copyto` end point to extract data bypasses the usual JSON formatting applied by the SQL API, so it can dump more data, faster. However, it has the restriction that it will only output in a handful of formats:
 
 * PgSQL [text format](https://www.postgresql.org/docs/10/static/sql-copy.html#id-1.9.3.52.9.2),

From 5f9cc37dba8e5b0bc569aedd9bd37f2cdefbb4a0 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Fri, 27 Apr 2018 12:19:53 +0200
Subject: [PATCH 016/133] Jshint change, DRY the content-disposition handler

---
 app/controllers/copy_controller.js | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 573ba5c2..2752bae6 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -113,14 +113,12 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
             res.on('error', next);
             pgstream.on('error', next);
             pgstream.on('end', cb);
-            if (_.isString(filename)) {
-                var contentDisposition = "attachment; filename=" + encodeURIComponent(filename);
-                res.setHeader("Content-Disposition", contentDisposition);
-            } else {
+            // User did not provide a preferred download filename
+            if (! _.isString(filename)) {
                 filename = 'carto-sql-copyto.dmp';
-                var contentDisposition = "attachment; filename=" + encodeURIComponent(filename);
-                res.setHeader("Content-Disposition", contentDisposition);
             }
+            var contentDisposition = "attachment; filename=" + encodeURIComponent(filename);
+            res.setHeader("Content-Disposition", contentDisposition);
             res.setHeader("Content-Type", "application/octet-stream");
             pgstream.pipe(res);
         });

From e2d88963073ec788c5fb028f8fa7fb4946e12a5a Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Fri, 27 Apr 2018 12:32:47 +0200
Subject: [PATCH 017/133] Typo in curl example

---
 app/controllers/copy_controller.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 2752bae6..3851720c 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -138,7 +138,7 @@ CopyController.prototype.handleCopyFrom = function (req, res) {
     // The storage engine writes the rowCount into req when it's 
     // finished. Hopefully any errors just propogate up.
     
-    // curl --form sql="COPY foo FROM STDOUT" http://cdb.localhost.lan:8080/api/v2/copyfrom --form file=@copyfrom.txt 
+    // curl --form sql="COPY foo FROM STDOUT" --form file=@copyfrom.txt http://cdb.localhost.lan:8080/api/v2/copyfrom 
 
 
     if (typeof req.file === "undefined") {

From 97d359498b2dcb6f03735cfa96da55f7e4bf5fae Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Fri, 27 Apr 2018 15:50:42 +0200
Subject: [PATCH 018/133] Add some explanation in the "copy to" examples

---
 doc/copy_queries.md | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 4fc15f99..817eb4e2 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -132,12 +132,26 @@ Using the `copyto` end point to extract data bypasses the usual JSON formatting
 
 ### CURL Example
 
+The SQL to start a "copy to" can specify
+
+* a table to read,
+* a table and subset of columns to read, or
+* an arbitrary SQL query to execute and read.
+
+For our example, we'll read back just the three columns we originally loaded:
+
+    COPY upload_example TO stdout WITH (FORMAT csv, HEADER true)
+    
+The SQL needs to be URL-encoded before being embedded in the CURL command, so the final result looks like this:
+
     curl \
         --output upload_example_dl.csv \
         "http://{username}.carto.com/api/v2/copyto?sql=COPY+upload_example+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
 
 ### Python Example
 
+The Python to "copy to" is very simple, because the HTTP call is a simple get. The only complexity in this example is at the end, where the result is streamed back block-by-block, to avoid pulling the entire download into memory before writing to file.
+
     import requests
     import re
 
@@ -148,7 +162,7 @@ Using the `copyto` end point to extract data bypasses the usual JSON formatting
 
     # request the download, specifying desired file name
     url = "http://%s.carto.com/api/v2/copyto" % username    
-    r = requests.get(url, params={'api_key':api_key, 'sql':sql, 'filename':download_file})
+    r = requests.get(url, params={'api_key':api_key, 'sql':sql, 'filename':download_file}, stream=True)
     r.raise_for_status()
 
     # read save file name from response headers

From 025438872737bc6fc28f194d72edfe7d13c85e44 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Fri, 27 Apr 2018 15:55:34 +0200
Subject: [PATCH 019/133] Clarify parameters for COPY success

---
 doc/copy_queries.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 817eb4e2..a7755c0c 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -14,7 +14,7 @@ The PostgreSQL `COPY` command is extremely fast, but requires very precise input
 * A `COPY` command that describes the table and columns of the upload file, and the format of the file.
 * An upload file that exactly matches the `COPY` command.
 
-If the `COPY` command and the target table do not match, the upload will fail.
+If the `COPY` command, the supplied file, and the target table do not all match, the upload will fail.
 
 "Copy from" copies data "from" your file, "to" CARTO. "Copy from" uses [multipart/form-data](https://stackoverflow.com/questions/8659808/how-does-http-file-upload-work) to stream an upload file to the server. This avoids limitations around file size and any need for temporary storage: the data travels from your file straight into the database.
 

From 8b3fea21302fce487b259f1459a304d067b4db0d Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey+github@cleverelephant.ca>
Date: Sat, 28 Apr 2018 19:28:57 -0400
Subject: [PATCH 020/133] Sync SQL to text

---
 doc/copy_queries.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index a7755c0c..81c9867b 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -140,13 +140,13 @@ The SQL to start a "copy to" can specify
 
 For our example, we'll read back just the three columns we originally loaded:
 
-    COPY upload_example TO stdout WITH (FORMAT csv, HEADER true)
+    COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)
     
 The SQL needs to be URL-encoded before being embedded in the CURL command, so the final result looks like this:
 
     curl \
         --output upload_example_dl.csv \
-        "http://{username}.carto.com/api/v2/copyto?sql=COPY+upload_example+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
+        "http://{username}.carto.com/api/v2/copyto?sql=COPY+upload_example+(the_geom,name,age)+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
 
 ### Python Example
 

From af4c5906c85584f988d133ba52cfbfb0fc3d5260 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Thu, 3 May 2018 18:31:49 +0200
Subject: [PATCH 021/133] adding /sql tro the endpoints

---
 app/controllers/copy_controller.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 3851720c..90260127 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -76,8 +76,8 @@ CopyController.prototype.route = function (app) {
         ];
     };
 
-    app.post(`${base_url}/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
-    app.get(`${base_url}/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
+    app.post(`${base_url}/sql/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
+    app.get(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
 };
 
 CopyController.prototype.copyDbParamsToReq = function (req, res, next) {
@@ -153,4 +153,4 @@ CopyController.prototype.handleCopyFrom = function (req, res) {
     
 };
 
-module.exports = CopyController;
\ No newline at end of file
+module.exports = CopyController;

From 7e71f8dc3bea5c561b326657027af5e217c336b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Thu, 3 May 2018 18:32:15 +0200
Subject: [PATCH 022/133] supporting multipart requests

---
 test/support/assert.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/support/assert.js b/test/support/assert.js
index 47c2ae51..3541f278 100644
--- a/test/support/assert.js
+++ b/test/support/assert.js
@@ -48,6 +48,10 @@ assert.response = function(server, req, res, callback) {
             requestParams.body = req.body || req.data;
         }
 
+        if (req.formData) {
+            requestParams.formData = req.formData;
+        }
+
         debug('Request params', requestParams);
         request(requestParams, function assert$response$requestHandler(error, response, body) {
             debug('Request response', error);

From 41a585d7612c3de7e1264039668d6187a7e6faa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Thu, 3 May 2018 18:32:39 +0200
Subject: [PATCH 023/133] test of copyfrom

---
 test/acceptance/copy-endpoints.js    | 25 +++++++++++++++++++++++++
 test/support/csv/copy_test_table.csv |  7 +++++++
 test/support/sql/test.sql            |  9 +++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 test/acceptance/copy-endpoints.js
 create mode 100644 test/support/csv/copy_test_table.csv

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
new file mode 100644
index 00000000..f07bb844
--- /dev/null
+++ b/test/acceptance/copy-endpoints.js
@@ -0,0 +1,25 @@
+require('../helper');
+
+const fs = require('fs');
+const server = require('../../app/server')();
+const assert = require('../support/assert');
+
+describe.only('copy-endpoints', function() {
+    it('should works with copyfrom endpoint', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql/copyfrom",
+            formData: {
+                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+            },
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            const response = JSON.parse(res.body);
+            assert.equal(!!response.time, true);
+            assert.strictEqual(response.total_rows, 6);
+            done();
+        });
+    });
+});
diff --git a/test/support/csv/copy_test_table.csv b/test/support/csv/copy_test_table.csv
new file mode 100644
index 00000000..79d2f3e7
--- /dev/null
+++ b/test/support/csv/copy_test_table.csv
@@ -0,0 +1,7 @@
+id,name
+11,Paul
+12,Peter
+13,Matthew
+14,
+15,James
+16,John
diff --git a/test/support/sql/test.sql b/test/support/sql/test.sql
index 8f8f9316..8a3b3459 100644
--- a/test/support/sql/test.sql
+++ b/test/support/sql/test.sql
@@ -217,3 +217,12 @@ INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('scoped_table_1'::re
 
 GRANT SELECT ON CDB_TableMetadata TO :TESTUSER;
 GRANT SELECT ON CDB_TableMetadata TO test_cartodb_user_2;
+
+DROP TABLE IF EXISTS copy_test;
+CREATE TABLE copy_test (
+    id integer,
+    name text,
+    age integer default 10
+);
+GRANT ALL ON TABLE copy_test TO :TESTUSER;
+GRANT ALL ON TABLE copy_test TO :PUBLICUSER;

From d19d236d241d82c9a41be7038c592f704307d42d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Thu, 3 May 2018 18:46:16 +0200
Subject: [PATCH 024/133] test of copyto

---
 test/acceptance/copy-endpoints.js | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index f07bb844..03290509 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -1,10 +1,11 @@
 require('../helper');
 
 const fs = require('fs');
+const querystring = require('querystring');
 const server = require('../../app/server')();
 const assert = require('../support/assert');
 
-describe.only('copy-endpoints', function() {
+describe('copy-endpoints', function() {
     it('should works with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom",
@@ -22,4 +23,23 @@ describe.only('copy-endpoints', function() {
             done();
         });
     });
+
+    it('should works with copyto endpoint', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql/copyto?" + querystring.stringify({
+                sql: 'COPY copy_test TO STDOUT',
+                filename: '/tmp/output.dmp'
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            assert.strictEqual(
+                res.body, 
+                '11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\tJohn\t10\n'
+            );
+            done();
+        });
+    });
+
 });

From 4322ccdd09e14075aca9f89b2f1c7076bc81320d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Thu, 3 May 2018 18:50:13 +0200
Subject: [PATCH 025/133] adding new endpoints to rate limits

---
 app/controllers/copy_controller.js | 4 ++--
 app/middlewares/rate-limit.js      | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 90260127..5ecbfff1 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -76,8 +76,8 @@ CopyController.prototype.route = function (app) {
         ];
     };
 
-    app.post(`${base_url}/sql/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
-    app.get(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.QUERY));
+    app.post(`${base_url}/sql/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM));
+    app.get(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO));
 };
 
 CopyController.prototype.copyDbParamsToReq = function (req, res, next) {
diff --git a/app/middlewares/rate-limit.js b/app/middlewares/rate-limit.js
index 3e4d2f47..4bf7f4c2 100644
--- a/app/middlewares/rate-limit.js
+++ b/app/middlewares/rate-limit.js
@@ -5,7 +5,9 @@ const RATE_LIMIT_ENDPOINTS_GROUPS = {
     QUERY_FORMAT: 'query_format',
     JOB_CREATE: 'job_create',
     JOB_GET: 'job_get',
-    JOB_DELETE: 'job_delete'
+    JOB_DELETE: 'job_delete',
+    COPY_FROM: 'copy_from',
+    COPY_TO: 'copy_to'
 };
 
 

From db3984021c813aea6bebed1ad23406317aa9b412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 10:50:14 +0200
Subject: [PATCH 026/133] ensuring query via multipart

---
 test/acceptance/query-multipart.js | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 test/acceptance/query-multipart.js

diff --git a/test/acceptance/query-multipart.js b/test/acceptance/query-multipart.js
new file mode 100644
index 00000000..300ecde1
--- /dev/null
+++ b/test/acceptance/query-multipart.js
@@ -0,0 +1,25 @@
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+
+describe('query-multipart', function() {
+    it('multipart form select', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql',
+            formData: {
+                q: 'SELECT 2 as n'
+            },
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            const response = JSON.parse(res.body);
+            assert.equal(!!response.time, true);
+            assert.strictEqual(response.total_rows, 1);
+            assert.deepStrictEqual(response.rows, [{n:2}]);
+            done();
+        });
+    });
+});

From 0346cf11d8c6eabc1a91c2c27340a55ae1cd81e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 15:11:45 +0200
Subject: [PATCH 027/133] enabling multipart/form-data again

---
 app/middlewares/body-parser.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/middlewares/body-parser.js b/app/middlewares/body-parser.js
index 7bfb8bf9..6c44e295 100644
--- a/app/middlewares/body-parser.js
+++ b/app/middlewares/body-parser.js
@@ -11,7 +11,7 @@
  */
 
 var qs = require('qs');
-//var multer = require('multer');
+var multer = require('multer');
 
 /**
  * Extract the mime type from the given request's
@@ -141,5 +141,5 @@ exports.parse['application/json'] = function(req, options, fn){
     });
 };
 
-// var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
-// exports.parse['multipart/form-data'] = multipartMiddleware.none();
+var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
+exports.parse['multipart/form-data'] = multipartMiddleware.none();

From 02238fefe169480b23db2acad37954fa5a90ccee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 15:15:37 +0200
Subject: [PATCH 028/133] moving body-parser from global to routes of query and
 job

---
 app/controllers/job_controller.js   | 5 +++++
 app/controllers/query_controller.js | 2 ++
 app/server.js                       | 2 --
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/controllers/job_controller.js b/app/controllers/job_controller.js
index 85a98884..b82706b2 100644
--- a/app/controllers/job_controller.js
+++ b/app/controllers/job_controller.js
@@ -1,5 +1,6 @@
 const util = require('util');
 
+const bodyParserMiddleware = require('../middlewares/body-parser');
 const userMiddleware = require('../middlewares/user');
 const { initializeProfilerMiddleware, finishProfilerMiddleware } = require('../middlewares/profiler');
 const authorizationMiddleware = require('../middlewares/authorization');
@@ -30,21 +31,25 @@ JobController.prototype.route = function (app) {
 
     app.get(
         `${base_url}/jobs-wip`, 
+        bodyParserMiddleware(),
         listWorkInProgressJobs(this.jobService), 
         sendResponse(), 
         errorMiddleware()
     );
     app.post(
         `${base_url}/sql/job`, 
+        bodyParserMiddleware(),
         checkBodyPayloadSize(), 
         jobMiddlewares('create', createJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_CREATE)
     );
     app.get(
         `${base_url}/sql/job/:job_id`, 
+        bodyParserMiddleware(),
         jobMiddlewares('retrieve', getJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_GET)
     );
     app.delete(
         `${base_url}/sql/job/:job_id`, 
+        bodyParserMiddleware(),
         jobMiddlewares('cancel', cancelJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_DELETE)
     );
 };
diff --git a/app/controllers/query_controller.js b/app/controllers/query_controller.js
index 6e5ccd61..7293bea4 100644
--- a/app/controllers/query_controller.js
+++ b/app/controllers/query_controller.js
@@ -12,6 +12,7 @@ var formats = require('../models/formats');
 
 var sanitize_filename = require('../utils/filename_sanitizer');
 var getContentDisposition = require('../utils/content_disposition');
+const bodyParserMiddleware = require('../middlewares/body-parser');
 const userMiddleware = require('../middlewares/user');
 const errorMiddleware = require('../middlewares/error');
 const authorizationMiddleware = require('../middlewares/authorization');
@@ -35,6 +36,7 @@ QueryController.prototype.route = function (app) {
     const { base_url } = global.settings;
     const queryMiddlewares = endpointGroup => {
         return [
+            bodyParserMiddleware(),
             initializeProfilerMiddleware('query'),
             userMiddleware(),
             rateLimitsMiddleware(this.userLimitsService, endpointGroup),
diff --git a/app/server.js b/app/server.js
index ee262262..c94252bc 100644
--- a/app/server.js
+++ b/app/server.js
@@ -15,7 +15,6 @@
 //
 
 var express = require('express');
-var bodyParser = require('./middlewares/body-parser');
 var Profiler = require('./stats/profiler-proxy');
 var _ = require('underscore');
 var TableCacheFactory = require('./utils/table_cache_factory');
@@ -128,7 +127,6 @@ function App(statsClient) {
       });
     }
 
-    app.use(bodyParser());
     app.enable('jsonp callback');
     app.set("trust proxy", true);
     app.disable('x-powered-by');

From 6939b54ac4fd295da8cdcba1884343926f0e0530 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 15:27:56 +0200
Subject: [PATCH 029/133] jshint happy

---
 test/acceptance/query-multipart.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/acceptance/query-multipart.js b/test/acceptance/query-multipart.js
index 300ecde1..b25d93e1 100644
--- a/test/acceptance/query-multipart.js
+++ b/test/acceptance/query-multipart.js
@@ -1,8 +1,7 @@
 require('../helper');
 
-var server = require('../../app/server')();
-var assert = require('../support/assert');
-var querystring = require('querystring');
+const server = require('../../app/server')();
+const assert = require('../support/assert');
 
 describe('query-multipart', function() {
     it('multipart form select', function(done){

From 2b066eadfab68544738f84803b6c7175aaea23ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 15:35:23 +0200
Subject: [PATCH 030/133] removing unused dependencies

---
 app/controllers/copy_controller.js | 5 +----
 app/server.js                      | 2 --
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 5ecbfff1..264823c5 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -1,7 +1,6 @@
 'use strict';
 
 var _ = require('underscore');
-var CachedQueryTables = require('../services/cached-query-tables');
 
 const userMiddleware = require('../middlewares/user');
 const errorMiddleware = require('../middlewares/error');
@@ -36,11 +35,9 @@ var upload = multer({ storage: multerpgcopy() });
 // var uploadLimits = { fileSize: 1024*1024*1024, fields: 10, files: 1 };
 // var upload = multer({ storage: multer.diskStorage({}), limits: uploadLimits });
 
-function CopyController(metadataBackend, userDatabaseService, tableCache, statsd_client, userLimitsService) {
+function CopyController(metadataBackend, userDatabaseService, userLimitsService) {
     this.metadataBackend = metadataBackend;
-    this.statsd_client = statsd_client;
     this.userDatabaseService = userDatabaseService;
-    this.queryTables = new CachedQueryTables(tableCache);
     this.userLimitsService = userLimitsService;
 }
 
diff --git a/app/server.js b/app/server.js
index c94252bc..7505b337 100644
--- a/app/server.js
+++ b/app/server.js
@@ -165,8 +165,6 @@ function App(statsClient) {
     var copyController = new CopyController(
         metadataBackend, 
         userDatabaseService, 
-        tableCache, 
-        statsClient, 
         userLimitsService
     );
     copyController.route(app);

From fa0584e40aa1e12b3280d232489bb4c3c75e3bce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 16:47:02 +0200
Subject: [PATCH 031/133] adding error tests

---
 test/acceptance/copy-endpoints.js | 61 +++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 03290509..de4fb49a 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -24,6 +24,67 @@ describe('copy-endpoints', function() {
         });
     });
 
+    it('should fail with copyfrom endpoint and unexisting table', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql/copyfrom",
+            formData: {
+                sql: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+            },
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            assert.deepEqual(
+                JSON.parse(res.body), 
+                {
+                    error:['relation \"unexisting_table\" does not exist']
+                }
+            );
+            done();
+        });
+    });
+
+    it('should fail with copyfrom endpoint and without csv', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql/copyfrom",
+            formData: {
+                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+            },
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            assert.deepEqual(
+                JSON.parse(res.body), 
+                {
+                    error:['no rows copied']
+                }
+            );
+            done();
+        });
+    });
+
+    it('should fail with copyfrom endpoint and without sql', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql/copyfrom",
+            formData: {
+                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+            },
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            assert.deepEqual(
+                JSON.parse(res.body), 
+                {
+                    error:["Parameter 'sql' is missing, must be in URL or first field in POST"]
+                }
+            );
+            done();
+        });
+    });
+
     it('should works with copyto endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyto?" + querystring.stringify({

From 383b86cf9b46e50f4e165d610a49e0c95e674989 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 16:47:50 +0200
Subject: [PATCH 032/133] simplify sql paramter handler and removing underscore

---
 app/utils/multer-pg-copy.js | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
index ef931119..b7641983 100644
--- a/app/utils/multer-pg-copy.js
+++ b/app/utils/multer-pg-copy.js
@@ -2,7 +2,6 @@
 // https://github.com/expressjs/multer/blob/master/StorageEngine.md
 // for the contract. 
 
-var _ = require('underscore');
 var fs = require('fs');
 var copyFrom = require('pg-copy-streams').from;
 var PSQL = require('cartodb-psql');
@@ -11,27 +10,17 @@ function PgCopyCustomStorage (opts) {
     this.opts = opts || {};
 }
 
-// Pick the URL or the body SQL query, prefering the body query
-// if both are filled in (???)
-function getSqlParam(req) {
-    var b_sql = req.body.sql;
-    b_sql = (b_sql === "" || _.isUndefined(b_sql)) ? null : b_sql;
-    var q_sql = req.query.sql;
-    q_sql = (q_sql === "" || _.isUndefined(q_sql)) ? null : q_sql;
-    return b_sql || q_sql;
-}
-
 PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb) {
 
     // Hopefully the body-parser has extracted the 'sql' parameter
     // or the user has provided it on the URL line.
     // Otherwise, this will be a short trip, as we won't be able
     // to the pg-copy-streams SQL command
-    var sql = getSqlParam(req);
+    var sql = req.body.sql || req.query.sql;
   
     // Ensure SQL parameter is not missing
-    if (!_.isString(sql)) {
-        cb(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
+    if (!sql) {
+        return cb(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
     }
     
     // Only accept SQL that starts with 'COPY'

From 6b96032fcf8c891d080dff95718c8531311d04e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 4 May 2018 16:50:17 +0200
Subject: [PATCH 033/133] ensuring return error

---
 app/utils/multer-pg-copy.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
index b7641983..1424db32 100644
--- a/app/utils/multer-pg-copy.js
+++ b/app/utils/multer-pg-copy.js
@@ -25,14 +25,14 @@ PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb)
     
     // Only accept SQL that starts with 'COPY'
     if (!sql.toUpperCase().startsWith("COPY ")) {
-        cb(new Error("SQL must start with COPY"));
+        return cb(new Error("SQL must start with COPY"));
     }    
 
     // We expect the an earlier middleware to have 
     // set this by the time we are called via multer,
     // so this should never happen
     if (!req.userDbParams) {
-        cb(new Error("req.userDbParams is not set"));
+        return cb(new Error("Something was wrong"));
     }
 
     var copyFromStream = copyFrom(sql);

From 7a8dcf7503df385edf987b6bfcb43d61f78a8e16 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cartodb.com>
Date: Mon, 7 May 2018 08:15:00 -0700
Subject: [PATCH 034/133] Minor typo

---
 doc/copy_queries.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 81c9867b..34a6243f 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -32,7 +32,7 @@ For a table to be readable by CARTO, it must have a minimum of three columns wit
 * `the_geom`, a `geometry` in the ESPG:4326 projection (aka long/lat)
 * `the_geom_webmercator`, a `geometry` in the ESPG:3857 projection (aka web mercator)
 
-Creating a new CARTO table with all the right triggers and columns can is tricky, so here is an example:
+Creating a new CARTO table with all the right triggers and columns can be tricky, so here is an example:
 
     -- create the table using the *required* columns and a 
     -- couple more

From a2d0163ece1f9431da687f13978c99d9c07ce0be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 12:51:52 +0200
Subject: [PATCH 035/133] ensuring works with sql parameter in query string

---
 test/acceptance/copy-endpoints.js | 24 +++++++++++++++++++++---
 test/support/sql/test.sql         | 17 +++++++++++++----
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index de4fb49a..8c84c1ec 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -10,7 +10,25 @@ describe('copy-endpoints', function() {
         assert.response(server, {
             url: "/api/v1/sql/copyfrom",
             formData: {
-                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+            },
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            const response = JSON.parse(res.body);
+            assert.equal(!!response.time, true);
+            assert.strictEqual(response.total_rows, 6);
+            done();
+        });
+    });
+    
+    it('should works with copyfrom endpoint and SQL in querystring', function(done){
+        const sql = "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)";
+        assert.response(server, {
+            url: `/api/v1/sql/copyfrom?sql=${sql}`,
+            formData: {
                 file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             },
             headers: {host: 'vizzuality.cartodb.com'},
@@ -58,7 +76,7 @@ describe('copy-endpoints', function() {
             assert.deepEqual(
                 JSON.parse(res.body), 
                 {
-                    error:['no rows copied']
+                    error:['The file is missing']
                 }
             );
             done();
@@ -88,7 +106,7 @@ describe('copy-endpoints', function() {
     it('should works with copyto endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyto?" + querystring.stringify({
-                sql: 'COPY copy_test TO STDOUT',
+                sql: 'COPY copy_endpoints_test TO STDOUT',
                 filename: '/tmp/output.dmp'
             }),
             headers: {host: 'vizzuality.cartodb.com'},
diff --git a/test/support/sql/test.sql b/test/support/sql/test.sql
index 8a3b3459..711e6183 100644
--- a/test/support/sql/test.sql
+++ b/test/support/sql/test.sql
@@ -218,11 +218,20 @@ INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('scoped_table_1'::re
 GRANT SELECT ON CDB_TableMetadata TO :TESTUSER;
 GRANT SELECT ON CDB_TableMetadata TO test_cartodb_user_2;
 
-DROP TABLE IF EXISTS copy_test;
-CREATE TABLE copy_test (
+DROP TABLE IF EXISTS copy_endpoints_test;
+CREATE TABLE copy_endpoints_test (
     id integer,
     name text,
     age integer default 10
 );
-GRANT ALL ON TABLE copy_test TO :TESTUSER;
-GRANT ALL ON TABLE copy_test TO :PUBLICUSER;
+GRANT ALL ON TABLE copy_endpoints_test TO :TESTUSER;
+GRANT ALL ON TABLE copy_endpoints_test TO :PUBLICUSER;
+
+DROP TABLE IF EXISTS copy_endpoints_test2;
+CREATE TABLE copy_endpoints_test2 (
+    id integer,
+    name text,
+    age integer default 10
+);
+GRANT ALL ON TABLE copy_endpoints_test2 TO :TESTUSER;
+GRANT ALL ON TABLE copy_endpoints_test2 TO :PUBLICUSER;

From b036b876ff451fe0fb54d575e33958e42a340621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 12:52:33 +0200
Subject: [PATCH 036/133] using Busboy instead of multer

---
 app/controllers/copy_controller.js | 139 +++++++++++++++++------------
 app/utils/multer-pg-copy.js        |  75 ----------------
 2 files changed, 80 insertions(+), 134 deletions(-)
 delete mode 100644 app/utils/multer-pg-copy.js

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 264823c5..2ad16a84 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -10,30 +10,12 @@ const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
 const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
+const Busboy = require('busboy');
 
-// Database requirements
 var PSQL = require('cartodb-psql');
 var copyTo = require('pg-copy-streams').to;
+var copyFrom = require('pg-copy-streams').from;
 
-// We need NPM body-parser so we can use the multer and
-// still decode the urlencoded 'sql' parameter from
-// the POST body
-var bodyParser = require('body-parser'); // NPM body-parser
-
-// We need multer to support multi-part POST content
-var multer = require('multer');
-
-// The default multer storage engines (file/memory) don't
-// do what we need, which is pipe the multer read stream
-// straight into the pg-copy write stream, so we use
-// a custom storage engine
-var multerpgcopy = require('../utils/multer-pg-copy');
-var upload = multer({ storage: multerpgcopy() });
-
-// Store the uploaded file in the tmp directory, with limits on the
-// size of acceptable uploads
-// var uploadLimits = { fileSize: 1024*1024*1024, fields: 10, files: 1 };
-// var upload = multer({ storage: multer.diskStorage({}), limits: uploadLimits });
 
 function CopyController(metadataBackend, userDatabaseService, userLimitsService) {
     this.metadataBackend = metadataBackend;
@@ -43,7 +25,7 @@ function CopyController(metadataBackend, userDatabaseService, userLimitsService)
 
 CopyController.prototype.route = function (app) {
     const { base_url } = global.settings;
-        
+
     const copyFromMiddlewares = endpointGroup => {
         return [
             initializeProfilerMiddleware('copyfrom'),
@@ -52,14 +34,12 @@ CopyController.prototype.route = function (app) {
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
-            this.copyDbParamsToReq.bind(this),
-            bodyParser.urlencoded({ extended: true }),
-            upload.single('file'),
             this.handleCopyFrom.bind(this),
+            this.responseCopyFrom.bind(this),
             errorMiddleware()
         ];
     };
-    
+
     const copyToMiddlewares = endpointGroup => {
         return [
             initializeProfilerMiddleware('copyto'),
@@ -77,41 +57,33 @@ CopyController.prototype.route = function (app) {
     app.get(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO));
 };
 
-CopyController.prototype.copyDbParamsToReq = function (req, res, next) {
-
-    req.userDbParams = res.locals.userDbParams;
-    next();    
-};
-
 CopyController.prototype.handleCopyTo = function (req, res, next) {
-        
-    // curl "http://cdb.localhost.lan:8080/api/v2/copyto?sql=copy+foo+to+stdout&filename=output.dmp"
-        
+
     var sql = req.query.sql;
     var filename = req.query.filename;
     sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
-    
+
     // Ensure SQL parameter is not missing
     if (!_.isString(sql)) {
         throw new Error("Parameter 'sql' is missing");
     }
-        
+
     // Only accept SQL that starts with 'COPY'
     if (!sql.toUpperCase().startsWith("COPY ")) {
         throw new Error("SQL must start with COPY");
     }
-    
-    try {        
+
+    try {
         // Open pgsql COPY pipe and stream out to HTTP response
         var pg = new PSQL(res.locals.userDbParams);
-        pg.connect(function(err, client, cb) {
+        pg.connect(function (err, client, cb) {
             var copyToStream = copyTo(sql);
             var pgstream = client.query(copyToStream);
             res.on('error', next);
             pgstream.on('error', next);
             pgstream.on('end', cb);
             // User did not provide a preferred download filename
-            if (! _.isString(filename)) {
+            if (!_.isString(filename)) {
                 filename = 'carto-sql-copyto.dmp';
             }
             var contentDisposition = "attachment; filename=" + encodeURIComponent(filename);
@@ -122,32 +94,81 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
     } catch (err) {
         next(err);
     }
-    
+
 };
 
 
-// jshint maxcomplexity:21
-CopyController.prototype.handleCopyFrom = function (req, res) {
-
-    // All the action happens in multer, which reads the incoming
-    // file into a stream, and then hands it to the custom storage
-    // engine defined in multer-pg-copy.js.
-    // The storage engine writes the rowCount into req when it's 
-    // finished. Hopefully any errors just propogate up.
+CopyController.prototype.handleCopyFrom = function (req, res, next) {
+    const busboy = new Busboy({ headers: req.headers });
+    let sql = req.query.sql;
+    let files = 0;
     
-    // curl --form sql="COPY foo FROM STDOUT" --form file=@copyfrom.txt http://cdb.localhost.lan:8080/api/v2/copyfrom 
+    busboy.on('field', function (fieldname, val) {
+        if (fieldname === 'sql') {
+            sql = val;
+            
+            // Only accept SQL that starts with 'COPY'
+            if (!sql.toUpperCase().startsWith("COPY ")) {
+                return next(new Error("SQL must start with COPY"));
+            }
+        }
+    });
 
+    busboy.on('file', function (fieldname, file) {
+        files++;
 
-    if (typeof req.file === "undefined") {
-        throw new Error("no rows copied");
-    }
-    var msg = {time: req.file.time, total_rows: req.file.total_rows};
-    if (req.query && req.query.callback) {
-        res.jsonp(msg);
-    } else {
-        res.json(msg);
-    }
+        if (!sql) {
+            return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
+        }
+        
+        try {
+            const start_time = Date.now();
+            
+            // Connect and run the COPY
+            const pg = new PSQL(res.locals.userDbParams);
+            pg.connect(function(err, client) {
+                if (err) {
+                    return next(err);
+                }
     
+                let copyFromStream = copyFrom(sql);
+                const pgstream = client.query(copyFromStream);
+                pgstream.on('error', next);
+                pgstream.on('end', function () {
+                    var end_time = Date.now();
+                    res.body = {
+                        time: (end_time - start_time)/1000,
+                        total_rows: copyFromStream.rowCount
+                    };
+
+                    return next();
+                });
+
+                file.pipe(pgstream);
+            });
+    
+        } catch (err) {
+            next(err);
+        }
+    });
+
+    busboy.on('finish', () => {
+        if(files !== 1) {
+            return next(new Error("The file is missing"));
+        }
+    });
+
+    busboy.on('error', next);
+
+    req.pipe(busboy);
+};
+
+CopyController.prototype.responseCopyFrom = function (req, res, next) {
+    if (!res.body || !res.body.total_rows) {
+        return next(new Error("No rows copied"));
+    }
+
+    res.send(res.body);
 };
 
 module.exports = CopyController;
diff --git a/app/utils/multer-pg-copy.js b/app/utils/multer-pg-copy.js
deleted file mode 100644
index 1424db32..00000000
--- a/app/utils/multer-pg-copy.js
+++ /dev/null
@@ -1,75 +0,0 @@
-// This is a multer "custom storage engine", see
-// https://github.com/expressjs/multer/blob/master/StorageEngine.md
-// for the contract. 
-
-var fs = require('fs');
-var copyFrom = require('pg-copy-streams').from;
-var PSQL = require('cartodb-psql');
-
-function PgCopyCustomStorage (opts) {
-    this.opts = opts || {};
-}
-
-PgCopyCustomStorage.prototype._handleFile = function _handleFile (req, file, cb) {
-
-    // Hopefully the body-parser has extracted the 'sql' parameter
-    // or the user has provided it on the URL line.
-    // Otherwise, this will be a short trip, as we won't be able
-    // to the pg-copy-streams SQL command
-    var sql = req.body.sql || req.query.sql;
-  
-    // Ensure SQL parameter is not missing
-    if (!sql) {
-        return cb(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
-    }
-    
-    // Only accept SQL that starts with 'COPY'
-    if (!sql.toUpperCase().startsWith("COPY ")) {
-        return cb(new Error("SQL must start with COPY"));
-    }    
-
-    // We expect the an earlier middleware to have 
-    // set this by the time we are called via multer,
-    // so this should never happen
-    if (!req.userDbParams) {
-        return cb(new Error("Something was wrong"));
-    }
-
-    var copyFromStream = copyFrom(sql);
-
-    try {
-        // Connect and run the COPY
-        var pg = new PSQL(req.userDbParams);
-        var start_time = Date.now();
-        
-        pg.connect(function(err, client, done) {
-            if (err) {
-                return done(err);
-            }
-            var pgstream = client.query(copyFromStream);
-            file.stream.on('error', cb);
-            pgstream.on('error', cb);
-            pgstream.on('end', function () {
-                var end_time = Date.now();
-                cb(null, {
-                    total_rows: copyFromStream.rowCount,
-                    time: (end_time - start_time)/1000
-                });
-            });
-            file.stream.pipe(pgstream);
-        });
-
-    } catch (err) {
-        cb(err);
-    }
-    return;
-
-};
-
-PgCopyCustomStorage.prototype._removeFile = function _removeFile (req, file, cb) {
-    fs.unlink(file.path, cb);
-};
-
-module.exports = function (opts) {
-    return new PgCopyCustomStorage(opts);
-};

From c40ab801b17563a722e87ae2abfe9ba095e9b2f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 13:08:29 +0200
Subject: [PATCH 037/133] refactoring handleCopyTo

---
 app/controllers/copy_controller.js | 61 ++++++++++++++++--------------
 1 file changed, 33 insertions(+), 28 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 2ad16a84..7310ee34 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -1,7 +1,5 @@
 'use strict';
 
-var _ = require('underscore');
-
 const userMiddleware = require('../middlewares/user');
 const errorMiddleware = require('../middlewares/error');
 const authorizationMiddleware = require('../middlewares/authorization');
@@ -49,6 +47,7 @@ CopyController.prototype.route = function (app) {
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
             this.handleCopyTo.bind(this),
+            this.responseCopyTo.bind(this),
             errorMiddleware()
         ];
     };
@@ -58,13 +57,9 @@ CopyController.prototype.route = function (app) {
 };
 
 CopyController.prototype.handleCopyTo = function (req, res, next) {
+    const { sql } = req.query;
 
-    var sql = req.query.sql;
-    var filename = req.query.filename;
-    sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
-
-    // Ensure SQL parameter is not missing
-    if (!_.isString(sql)) {
+    if (!sql) {
         throw new Error("Parameter 'sql' is missing");
     }
 
@@ -76,19 +71,18 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
     try {
         // Open pgsql COPY pipe and stream out to HTTP response
         var pg = new PSQL(res.locals.userDbParams);
-        pg.connect(function (err, client, cb) {
-            var copyToStream = copyTo(sql);
-            var pgstream = client.query(copyToStream);
+        pg.connect(function (err, client) {
+            if (err) {
+                return next(err);
+            }
+
+            let copyToStream = copyTo(sql);
+            const pgstream = client.query(copyToStream);
+
             res.on('error', next);
             pgstream.on('error', next);
-            pgstream.on('end', cb);
-            // User did not provide a preferred download filename
-            if (!_.isString(filename)) {
-                filename = 'carto-sql-copyto.dmp';
-            }
-            var contentDisposition = "attachment; filename=" + encodeURIComponent(filename);
-            res.setHeader("Content-Disposition", contentDisposition);
-            res.setHeader("Content-Type", "application/octet-stream");
+            pgstream.on('end', next);
+            
             pgstream.pipe(res);
         });
     } catch (err) {
@@ -97,16 +91,27 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
 
 };
 
+CopyController.prototype.responseCopyTo = function (req, res) {
+    let { filename } = req.query;
+    
+    if (!filename) {
+        filename = 'carto-sql-copyto.dmp';
+    }
+
+    res.setHeader("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
+    res.setHeader("Content-Type", "application/octet-stream");
+    res.send();
+};
 
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
     const busboy = new Busboy({ headers: req.headers });
     let sql = req.query.sql;
     let files = 0;
-    
+
     busboy.on('field', function (fieldname, val) {
         if (fieldname === 'sql') {
             sql = val;
-            
+
             // Only accept SQL that starts with 'COPY'
             if (!sql.toUpperCase().startsWith("COPY ")) {
                 return next(new Error("SQL must start with COPY"));
@@ -120,24 +125,24 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
         if (!sql) {
             return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
         }
-        
+
         try {
             const start_time = Date.now();
-            
+
             // Connect and run the COPY
             const pg = new PSQL(res.locals.userDbParams);
-            pg.connect(function(err, client) {
+            pg.connect(function (err, client) {
                 if (err) {
                     return next(err);
                 }
-    
+
                 let copyFromStream = copyFrom(sql);
                 const pgstream = client.query(copyFromStream);
                 pgstream.on('error', next);
                 pgstream.on('end', function () {
                     var end_time = Date.now();
                     res.body = {
-                        time: (end_time - start_time)/1000,
+                        time: (end_time - start_time) / 1000,
                         total_rows: copyFromStream.rowCount
                     };
 
@@ -146,14 +151,14 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
 
                 file.pipe(pgstream);
             });
-    
+
         } catch (err) {
             next(err);
         }
     });
 
     busboy.on('finish', () => {
-        if(files !== 1) {
+        if (files !== 1) {
             return next(new Error("The file is missing"));
         }
     });

From 7c4409bcf54a5407d38b12fc6f4f0b4ed309aae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 13:10:25 +0200
Subject: [PATCH 038/133] from var to const/let

---
 app/controllers/copy_controller.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 7310ee34..0fc8260b 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -10,9 +10,9 @@ const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 const Busboy = require('busboy');
 
-var PSQL = require('cartodb-psql');
-var copyTo = require('pg-copy-streams').to;
-var copyFrom = require('pg-copy-streams').from;
+const PSQL = require('cartodb-psql');
+const copyTo = require('pg-copy-streams').to;
+const copyFrom = require('pg-copy-streams').from;
 
 
 function CopyController(metadataBackend, userDatabaseService, userLimitsService) {
@@ -70,7 +70,7 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
 
     try {
         // Open pgsql COPY pipe and stream out to HTTP response
-        var pg = new PSQL(res.locals.userDbParams);
+        const pg = new PSQL(res.locals.userDbParams);
         pg.connect(function (err, client) {
             if (err) {
                 return next(err);
@@ -140,7 +140,7 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
                 const pgstream = client.query(copyFromStream);
                 pgstream.on('error', next);
                 pgstream.on('end', function () {
-                    var end_time = Date.now();
+                    const end_time = Date.now();
                     res.body = {
                         time: (end_time - start_time) / 1000,
                         total_rows: copyFromStream.rowCount

From 82d14ab98f47594445bee7cf4f8c19dffc664055 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 13:15:19 +0200
Subject: [PATCH 039/133] busboy configuration

---
 app/controllers/copy_controller.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 0fc8260b..474edb9b 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -104,10 +104,17 @@ CopyController.prototype.responseCopyTo = function (req, res) {
 };
 
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
-    const busboy = new Busboy({ headers: req.headers });
     let sql = req.query.sql;
     let files = 0;
 
+    const busboy = new Busboy({ 
+        headers: req.headers,
+        limits: {
+            fieldSize: Infinity,
+            files: 1
+        }
+    });
+
     busboy.on('field', function (fieldname, val) {
         if (fieldname === 'sql') {
             sql = val;

From 40c7878da9e72478347ac5da8d2bb791e6b82e01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 18:38:57 +0200
Subject: [PATCH 040/133] improve test message

---
 test/acceptance/query-multipart.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/acceptance/query-multipart.js b/test/acceptance/query-multipart.js
index b25d93e1..cb88870e 100644
--- a/test/acceptance/query-multipart.js
+++ b/test/acceptance/query-multipart.js
@@ -4,7 +4,7 @@ const server = require('../../app/server')();
 const assert = require('../support/assert');
 
 describe('query-multipart', function() {
-    it('multipart form select', function(done){
+    it('make query from a multipart form', function(done){
         assert.response(server, {
             url: '/api/v1/sql',
             formData: {

From d400926387baa792d1f5bc5c48756bcb1627f86a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 18:39:16 +0200
Subject: [PATCH 041/133] using busboy instead of multer

---
 app/middlewares/body-parser.js | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/app/middlewares/body-parser.js b/app/middlewares/body-parser.js
index 6c44e295..dd704d0e 100644
--- a/app/middlewares/body-parser.js
+++ b/app/middlewares/body-parser.js
@@ -11,7 +11,7 @@
  */
 
 var qs = require('qs');
-var multer = require('multer');
+const Busboy = require('busboy');
 
 /**
  * Extract the mime type from the given request's
@@ -141,5 +141,26 @@ exports.parse['application/json'] = function(req, options, fn){
     });
 };
 
-var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
-exports.parse['multipart/form-data'] = multipartMiddleware.none();
+
+exports.parse['multipart/form-data'] = function(req, options, fn){
+    if (req.method === 'POST') {
+        req.setEncoding('utf8');
+
+        const busboy = new Busboy({ 
+            headers: req.headers,
+            limits: {
+                files: 0
+            }
+        });
+
+        busboy.on('field', function(fieldname, val) {
+            req.body[fieldname] = val;
+        });
+
+        busboy.on('finish', function() {
+            fn();
+        });
+
+        req.pipe(busboy);
+    }
+};

From 7109b204e2efb1ed9c6ecd1bb770c36ee3d974f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 18:44:51 +0200
Subject: [PATCH 042/133] awaiting for fuc**** shrinkwrap

---
 npm-shrinkwrap.json | 1340 -------------------------------------------
 package.json        |    1 -
 2 files changed, 1341 deletions(-)
 delete mode 100644 npm-shrinkwrap.json

diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
deleted file mode 100644
index 587d750d..00000000
--- a/npm-shrinkwrap.json
+++ /dev/null
@@ -1,1340 +0,0 @@
-{
-  "name": "cartodb_sql_api",
-  "version": "2.0.1",
-  "dependencies": {
-    "accepts": {
-      "version": "1.2.13",
-      "from": "accepts@>=1.2.12 <1.3.0",
-      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz"
-    },
-    "ansi-regex": {
-      "version": "2.1.1",
-      "from": "ansi-regex@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz"
-    },
-    "ansi-styles": {
-      "version": "2.2.1",
-      "from": "ansi-styles@>=2.2.1 <3.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz"
-    },
-    "append-field": {
-      "version": "0.1.0",
-      "from": "append-field@>=0.1.0 <0.2.0",
-      "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz"
-    },
-    "array-flatten": {
-      "version": "1.1.1",
-      "from": "array-flatten@1.1.1",
-      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
-    },
-    "asn1": {
-      "version": "0.2.3",
-      "from": "asn1@>=0.2.3 <0.3.0",
-      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz"
-    },
-    "assert-plus": {
-      "version": "0.2.0",
-      "from": "assert-plus@>=0.2.0 <0.3.0",
-      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz"
-    },
-    "async": {
-      "version": "0.2.10",
-      "from": "async@>=0.2.0 <0.3.0",
-      "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz"
-    },
-    "asynckit": {
-      "version": "0.4.0",
-      "from": "asynckit@>=0.4.0 <0.5.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz"
-    },
-    "aws-sign2": {
-      "version": "0.6.0",
-      "from": "aws-sign2@>=0.6.0 <0.7.0",
-      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
-    },
-    "aws4": {
-      "version": "1.7.0",
-      "from": "aws4@>=1.2.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz"
-    },
-    "balanced-match": {
-      "version": "1.0.0",
-      "from": "balanced-match@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz"
-    },
-    "basic-auth": {
-      "version": "2.0.0",
-      "from": "basic-auth@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.0.tgz"
-    },
-    "bcrypt-pbkdf": {
-      "version": "1.0.1",
-      "from": "bcrypt-pbkdf@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
-      "optional": true
-    },
-    "bindings": {
-      "version": "1.3.0",
-      "from": "bindings@>=1.2.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz"
-    },
-    "bintrees": {
-      "version": "1.0.1",
-      "from": "bintrees@1.0.1",
-      "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz"
-    },
-    "bl": {
-      "version": "1.1.2",
-      "from": "bl@>=1.1.2 <1.2.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz",
-      "dependencies": {
-        "isarray": {
-          "version": "1.0.0",
-          "from": "isarray@>=1.0.0 <1.1.0",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
-        },
-        "process-nextick-args": {
-          "version": "1.0.7",
-          "from": "process-nextick-args@>=1.0.6 <1.1.0",
-          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz"
-        },
-        "readable-stream": {
-          "version": "2.0.6",
-          "from": "readable-stream@>=2.0.5 <2.1.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz"
-        }
-      }
-    },
-    "bluebird": {
-      "version": "3.5.1",
-      "from": "bluebird@>=3.3.3 <4.0.0",
-      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz"
-    },
-    "body-parser": {
-      "version": "1.18.2",
-      "from": "body-parser@1.18.2",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
-      "dependencies": {
-        "debug": {
-          "version": "2.6.9",
-          "from": "debug@2.6.9",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz"
-        },
-        "qs": {
-          "version": "6.5.1",
-          "from": "qs@6.5.1",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz"
-        }
-      }
-    },
-    "boom": {
-      "version": "2.10.1",
-      "from": "boom@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
-    },
-    "brace-expansion": {
-      "version": "1.1.11",
-      "from": "brace-expansion@>=1.1.7 <2.0.0",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz"
-    },
-    "buffer-from": {
-      "version": "1.0.0",
-      "from": "buffer-from@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.0.0.tgz"
-    },
-    "buffer-writer": {
-      "version": "1.0.1",
-      "from": "buffer-writer@1.0.1",
-      "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz"
-    },
-    "builtin-modules": {
-      "version": "1.1.1",
-      "from": "builtin-modules@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz"
-    },
-    "bunyan": {
-      "version": "1.8.1",
-      "from": "bunyan@1.8.1",
-      "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.1.tgz"
-    },
-    "busboy": {
-      "version": "0.2.14",
-      "from": "busboy@>=0.2.11 <0.3.0",
-      "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
-      "dependencies": {
-        "readable-stream": {
-          "version": "1.1.14",
-          "from": "readable-stream@>=1.1.0 <1.2.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz"
-        }
-      }
-    },
-    "bytes": {
-      "version": "3.0.0",
-      "from": "bytes@3.0.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz"
-    },
-    "camelcase": {
-      "version": "3.0.0",
-      "from": "camelcase@>=3.0.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz"
-    },
-    "cartodb-psql": {
-      "version": "0.10.1",
-      "from": "cartodb-psql@0.10.1",
-      "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.10.1.tgz"
-    },
-    "cartodb-query-tables": {
-      "version": "0.2.0",
-      "from": "cartodb-query-tables@0.2.0",
-      "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz"
-    },
-    "cartodb-redis": {
-      "version": "1.0.0",
-      "from": "cartodb-redis@1.0.0",
-      "resolved": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-1.0.0.tgz"
-    },
-    "caseless": {
-      "version": "0.11.0",
-      "from": "caseless@>=0.11.0 <0.12.0",
-      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz"
-    },
-    "chalk": {
-      "version": "1.1.3",
-      "from": "chalk@>=1.1.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz"
-    },
-    "cliui": {
-      "version": "3.2.0",
-      "from": "cliui@>=3.2.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz"
-    },
-    "code-point-at": {
-      "version": "1.1.0",
-      "from": "code-point-at@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz"
-    },
-    "combined-stream": {
-      "version": "1.0.6",
-      "from": "combined-stream@>=1.0.5 <1.1.0",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz"
-    },
-    "commander": {
-      "version": "2.15.1",
-      "from": "commander@>=2.9.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz"
-    },
-    "concat-map": {
-      "version": "0.0.1",
-      "from": "concat-map@0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz"
-    },
-    "concat-stream": {
-      "version": "1.6.2",
-      "from": "concat-stream@>=1.5.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-      "dependencies": {
-        "isarray": {
-          "version": "1.0.0",
-          "from": "isarray@>=1.0.0 <1.1.0",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
-        },
-        "readable-stream": {
-          "version": "2.3.6",
-          "from": "readable-stream@>=2.2.2 <3.0.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz"
-        },
-        "string_decoder": {
-          "version": "1.1.1",
-          "from": "string_decoder@>=1.1.1 <1.2.0",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz"
-        }
-      }
-    },
-    "content-disposition": {
-      "version": "0.5.1",
-      "from": "content-disposition@0.5.1",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz"
-    },
-    "content-type": {
-      "version": "1.0.4",
-      "from": "content-type@>=1.0.4 <1.1.0",
-      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz"
-    },
-    "cookie": {
-      "version": "0.1.5",
-      "from": "cookie@0.1.5",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz"
-    },
-    "cookie-signature": {
-      "version": "1.0.6",
-      "from": "cookie-signature@1.0.6",
-      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
-    },
-    "core-util-is": {
-      "version": "1.0.2",
-      "from": "core-util-is@>=1.0.0 <1.1.0",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
-    },
-    "cryptiles": {
-      "version": "2.0.5",
-      "from": "cryptiles@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
-    },
-    "dashdash": {
-      "version": "1.14.1",
-      "from": "dashdash@>=1.12.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "from": "assert-plus@>=1.0.0 <2.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
-        }
-      }
-    },
-    "debug": {
-      "version": "2.2.0",
-      "from": "debug@2.2.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
-      "dependencies": {
-        "ms": {
-          "version": "0.7.1",
-          "from": "ms@0.7.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-        }
-      }
-    },
-    "decamelize": {
-      "version": "1.2.0",
-      "from": "decamelize@>=1.1.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz"
-    },
-    "delayed-stream": {
-      "version": "1.0.0",
-      "from": "delayed-stream@>=1.0.0 <1.1.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz"
-    },
-    "depd": {
-      "version": "1.1.2",
-      "from": "depd@>=1.1.1 <1.2.0",
-      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz"
-    },
-    "destroy": {
-      "version": "1.0.4",
-      "from": "destroy@>=1.0.4 <1.1.0",
-      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz"
-    },
-    "dicer": {
-      "version": "0.2.5",
-      "from": "dicer@0.2.5",
-      "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz",
-      "dependencies": {
-        "readable-stream": {
-          "version": "1.1.14",
-          "from": "readable-stream@>=1.1.0 <1.2.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz"
-        }
-      }
-    },
-    "dot": {
-      "version": "1.0.3",
-      "from": "dot@>=1.0.2 <1.1.0",
-      "resolved": "https://registry.npmjs.org/dot/-/dot-1.0.3.tgz"
-    },
-    "double-ended-queue": {
-      "version": "2.1.0-0",
-      "from": "double-ended-queue@>=2.1.0-0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz"
-    },
-    "dtrace-provider": {
-      "version": "0.6.0",
-      "from": "dtrace-provider@>=0.6.0 <0.7.0",
-      "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz",
-      "optional": true
-    },
-    "ecc-jsbn": {
-      "version": "0.1.1",
-      "from": "ecc-jsbn@>=0.1.1 <0.2.0",
-      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
-      "optional": true
-    },
-    "ee-first": {
-      "version": "1.1.1",
-      "from": "ee-first@1.1.1",
-      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
-    },
-    "error-ex": {
-      "version": "1.3.1",
-      "from": "error-ex@>=1.2.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz"
-    },
-    "escape-html": {
-      "version": "1.0.3",
-      "from": "escape-html@>=1.0.3 <1.1.0",
-      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz"
-    },
-    "escape-string-regexp": {
-      "version": "1.0.5",
-      "from": "escape-string-regexp@>=1.0.2 <2.0.0",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
-    },
-    "etag": {
-      "version": "1.7.0",
-      "from": "etag@>=1.7.0 <1.8.0",
-      "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
-    },
-    "express": {
-      "version": "4.13.4",
-      "from": "express@>=4.13.3 <4.14.0",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.13.4.tgz",
-      "dependencies": {
-        "qs": {
-          "version": "4.0.0",
-          "from": "qs@4.0.0",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
-        }
-      }
-    },
-    "extend": {
-      "version": "3.0.1",
-      "from": "extend@>=3.0.0 <3.1.0",
-      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz"
-    },
-    "extsprintf": {
-      "version": "1.3.0",
-      "from": "extsprintf@1.3.0",
-      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz"
-    },
-    "finalhandler": {
-      "version": "0.4.1",
-      "from": "finalhandler@0.4.1",
-      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz"
-    },
-    "find-up": {
-      "version": "1.1.2",
-      "from": "find-up@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz"
-    },
-    "forever-agent": {
-      "version": "0.6.1",
-      "from": "forever-agent@>=0.6.1 <0.7.0",
-      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
-    },
-    "form-data": {
-      "version": "2.0.0",
-      "from": "form-data@>=2.0.0 <2.1.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.0.0.tgz"
-    },
-    "forwarded": {
-      "version": "0.1.2",
-      "from": "forwarded@>=0.1.0 <0.2.0",
-      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz"
-    },
-    "fresh": {
-      "version": "0.3.0",
-      "from": "fresh@0.3.0",
-      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
-    },
-    "generate-function": {
-      "version": "2.0.0",
-      "from": "generate-function@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz"
-    },
-    "generate-object-property": {
-      "version": "1.2.0",
-      "from": "generate-object-property@>=1.1.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz"
-    },
-    "generic-pool": {
-      "version": "2.4.3",
-      "from": "generic-pool@2.4.3",
-      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.3.tgz"
-    },
-    "get-caller-file": {
-      "version": "1.0.2",
-      "from": "get-caller-file@>=1.0.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz"
-    },
-    "getpass": {
-      "version": "0.1.7",
-      "from": "getpass@>=0.1.1 <0.2.0",
-      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "from": "assert-plus@>=1.0.0 <2.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
-        }
-      }
-    },
-    "glob": {
-      "version": "6.0.4",
-      "from": "glob@>=6.0.1 <7.0.0",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
-      "optional": true
-    },
-    "graceful-fs": {
-      "version": "4.1.11",
-      "from": "graceful-fs@>=4.1.2 <5.0.0",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz"
-    },
-    "har-validator": {
-      "version": "2.0.6",
-      "from": "har-validator@>=2.0.6 <2.1.0",
-      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz"
-    },
-    "has-ansi": {
-      "version": "2.0.0",
-      "from": "has-ansi@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz"
-    },
-    "hawk": {
-      "version": "3.1.3",
-      "from": "hawk@>=3.1.3 <3.2.0",
-      "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz"
-    },
-    "hiredis": {
-      "version": "0.5.0",
-      "from": "hiredis@>=0.5.0 <0.6.0",
-      "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.5.0.tgz"
-    },
-    "hoek": {
-      "version": "2.16.3",
-      "from": "hoek@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
-    },
-    "hosted-git-info": {
-      "version": "2.6.0",
-      "from": "hosted-git-info@>=2.1.4 <3.0.0",
-      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.6.0.tgz"
-    },
-    "http-errors": {
-      "version": "1.6.3",
-      "from": "http-errors@>=1.6.2 <1.7.0",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz"
-    },
-    "http-signature": {
-      "version": "1.1.1",
-      "from": "http-signature@>=1.1.0 <1.2.0",
-      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz"
-    },
-    "iconv-lite": {
-      "version": "0.4.19",
-      "from": "iconv-lite@0.4.19",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz"
-    },
-    "inflight": {
-      "version": "1.0.6",
-      "from": "inflight@>=1.0.4 <2.0.0",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz"
-    },
-    "inherits": {
-      "version": "2.0.3",
-      "from": "inherits@2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
-    },
-    "invert-kv": {
-      "version": "1.0.0",
-      "from": "invert-kv@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz"
-    },
-    "ipaddr.js": {
-      "version": "1.0.5",
-      "from": "ipaddr.js@1.0.5",
-      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.5.tgz"
-    },
-    "is-arrayish": {
-      "version": "0.2.1",
-      "from": "is-arrayish@>=0.2.1 <0.3.0",
-      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz"
-    },
-    "is-builtin-module": {
-      "version": "1.0.0",
-      "from": "is-builtin-module@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz"
-    },
-    "is-fullwidth-code-point": {
-      "version": "1.0.0",
-      "from": "is-fullwidth-code-point@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz"
-    },
-    "is-my-ip-valid": {
-      "version": "1.0.0",
-      "from": "is-my-ip-valid@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-my-ip-valid/-/is-my-ip-valid-1.0.0.tgz"
-    },
-    "is-my-json-valid": {
-      "version": "2.17.2",
-      "from": "is-my-json-valid@>=2.12.4 <3.0.0",
-      "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.17.2.tgz"
-    },
-    "is-property": {
-      "version": "1.0.2",
-      "from": "is-property@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz"
-    },
-    "is-typedarray": {
-      "version": "1.0.0",
-      "from": "is-typedarray@>=1.0.0 <1.1.0",
-      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz"
-    },
-    "is-utf8": {
-      "version": "0.2.1",
-      "from": "is-utf8@>=0.2.0 <0.3.0",
-      "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz"
-    },
-    "isarray": {
-      "version": "0.0.1",
-      "from": "isarray@0.0.1",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
-    },
-    "isstream": {
-      "version": "0.1.2",
-      "from": "isstream@>=0.1.2 <0.2.0",
-      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
-    },
-    "js-string-escape": {
-      "version": "1.0.1",
-      "from": "js-string-escape@1.0.1",
-      "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz"
-    },
-    "jsbn": {
-      "version": "0.1.1",
-      "from": "jsbn@>=0.1.0 <0.2.0",
-      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
-      "optional": true
-    },
-    "json-schema": {
-      "version": "0.2.3",
-      "from": "json-schema@0.2.3",
-      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz"
-    },
-    "json-stringify-safe": {
-      "version": "5.0.1",
-      "from": "json-stringify-safe@>=5.0.1 <5.1.0",
-      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
-    },
-    "jsonpointer": {
-      "version": "4.0.1",
-      "from": "jsonpointer@>=4.0.0 <5.0.0",
-      "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.1.tgz"
-    },
-    "jsprim": {
-      "version": "1.4.1",
-      "from": "jsprim@>=1.2.2 <2.0.0",
-      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "from": "assert-plus@1.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
-        }
-      }
-    },
-    "lcid": {
-      "version": "1.0.0",
-      "from": "lcid@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz"
-    },
-    "load-json-file": {
-      "version": "1.1.0",
-      "from": "load-json-file@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz"
-    },
-    "lodash.assign": {
-      "version": "4.2.0",
-      "from": "lodash.assign@>=4.2.0 <5.0.0",
-      "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz"
-    },
-    "log4js": {
-      "version": "0.6.25",
-      "from": "cartodb/log4js-node#cdb",
-      "resolved": "git://github.com/cartodb/log4js-node.git#145d5f91e35e7fb14a6278cbf7a711ced6603727",
-      "dependencies": {
-        "semver": {
-          "version": "4.3.6",
-          "from": "semver@>=4.3.3 <4.4.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz"
-        },
-        "underscore": {
-          "version": "1.8.2",
-          "from": "underscore@1.8.2",
-          "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.2.tgz"
-        }
-      }
-    },
-    "lru-cache": {
-      "version": "2.5.2",
-      "from": "lru-cache@>=2.5.0 <2.6.0",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.2.tgz"
-    },
-    "media-typer": {
-      "version": "0.3.0",
-      "from": "media-typer@0.3.0",
-      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
-    },
-    "merge-descriptors": {
-      "version": "1.0.1",
-      "from": "merge-descriptors@1.0.1",
-      "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz"
-    },
-    "methods": {
-      "version": "1.1.2",
-      "from": "methods@>=1.1.2 <1.2.0",
-      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
-    },
-    "mime": {
-      "version": "1.3.4",
-      "from": "mime@1.3.4",
-      "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
-    },
-    "mime-db": {
-      "version": "1.33.0",
-      "from": "mime-db@>=1.33.0 <1.34.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz"
-    },
-    "mime-types": {
-      "version": "2.1.18",
-      "from": "mime-types@>=2.1.18 <2.2.0",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz"
-    },
-    "minimatch": {
-      "version": "3.0.4",
-      "from": "minimatch@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz"
-    },
-    "minimist": {
-      "version": "0.0.8",
-      "from": "minimist@0.0.8",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz"
-    },
-    "mkdirp": {
-      "version": "0.5.1",
-      "from": "mkdirp@>=0.5.1 <0.6.0",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz"
-    },
-    "moment": {
-      "version": "2.22.0",
-      "from": "moment@>=2.10.6 <3.0.0",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.0.tgz",
-      "optional": true
-    },
-    "ms": {
-      "version": "2.0.0",
-      "from": "ms@2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz"
-    },
-    "multer": {
-      "version": "1.2.1",
-      "from": "multer@>=1.2.0 <1.3.0",
-      "resolved": "https://registry.npmjs.org/multer/-/multer-1.2.1.tgz",
-      "dependencies": {
-        "object-assign": {
-          "version": "3.0.0",
-          "from": "object-assign@>=3.0.0 <4.0.0",
-          "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-3.0.0.tgz"
-        }
-      }
-    },
-    "mv": {
-      "version": "2.1.1",
-      "from": "mv@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
-      "optional": true
-    },
-    "nan": {
-      "version": "2.10.0",
-      "from": "nan@>=2.0.8 <3.0.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz"
-    },
-    "ncp": {
-      "version": "2.0.0",
-      "from": "ncp@>=2.0.0 <2.1.0",
-      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
-      "optional": true
-    },
-    "negotiator": {
-      "version": "0.5.3",
-      "from": "negotiator@0.5.3",
-      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
-    },
-    "node-statsd": {
-      "version": "0.0.7",
-      "from": "node-statsd@>=0.0.7 <0.1.0",
-      "resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.0.7.tgz"
-    },
-    "node-uuid": {
-      "version": "1.4.8",
-      "from": "node-uuid@>=1.4.7 <2.0.0",
-      "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz"
-    },
-    "normalize-package-data": {
-      "version": "2.4.0",
-      "from": "normalize-package-data@>=2.3.2 <3.0.0",
-      "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz"
-    },
-    "number-is-nan": {
-      "version": "1.0.1",
-      "from": "number-is-nan@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz"
-    },
-    "oauth-client": {
-      "version": "0.3.0",
-      "from": "oauth-client@0.3.0",
-      "resolved": "https://registry.npmjs.org/oauth-client/-/oauth-client-0.3.0.tgz",
-      "dependencies": {
-        "node-uuid": {
-          "version": "1.1.0",
-          "from": "node-uuid@1.1.0",
-          "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.1.0.tgz"
-        }
-      }
-    },
-    "oauth-sign": {
-      "version": "0.8.2",
-      "from": "oauth-sign@>=0.8.1 <0.9.0",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz"
-    },
-    "object-assign": {
-      "version": "4.1.0",
-      "from": "object-assign@4.1.0",
-      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.0.tgz"
-    },
-    "on-finished": {
-      "version": "2.3.0",
-      "from": "on-finished@>=2.3.0 <2.4.0",
-      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz"
-    },
-    "once": {
-      "version": "1.4.0",
-      "from": "once@>=1.3.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz"
-    },
-    "optimist": {
-      "version": "0.3.5",
-      "from": "optimist@0.3.5",
-      "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.5.tgz"
-    },
-    "os-locale": {
-      "version": "1.4.0",
-      "from": "os-locale@>=1.4.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz"
-    },
-    "packet-reader": {
-      "version": "0.2.0",
-      "from": "packet-reader@0.2.0",
-      "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.2.0.tgz"
-    },
-    "parse-json": {
-      "version": "2.2.0",
-      "from": "parse-json@>=2.2.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz"
-    },
-    "parseurl": {
-      "version": "1.3.2",
-      "from": "parseurl@>=1.3.1 <1.4.0",
-      "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz"
-    },
-    "path-exists": {
-      "version": "2.1.0",
-      "from": "path-exists@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz"
-    },
-    "path-is-absolute": {
-      "version": "1.0.1",
-      "from": "path-is-absolute@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz"
-    },
-    "path-to-regexp": {
-      "version": "0.1.7",
-      "from": "path-to-regexp@0.1.7",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
-    },
-    "path-type": {
-      "version": "1.1.0",
-      "from": "path-type@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz"
-    },
-    "pg": {
-      "version": "6.1.6",
-      "from": "cartodb/node-postgres#6.1.6-cdb1",
-      "resolved": "git://github.com/cartodb/node-postgres.git#3eef52dd1e655f658a4ee8ac5697688b3ecfed44"
-    },
-    "pg-connection-string": {
-      "version": "0.1.3",
-      "from": "pg-connection-string@0.1.3",
-      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz"
-    },
-    "pg-copy-streams": {
-      "version": "1.2.0",
-      "from": "pg-copy-streams@>=1.2.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-1.2.0.tgz"
-    },
-    "pg-int8": {
-      "version": "1.0.1",
-      "from": "pg-int8@1.0.1",
-      "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz"
-    },
-    "pg-pool": {
-      "version": "1.8.0",
-      "from": "pg-pool@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-1.8.0.tgz"
-    },
-    "pg-types": {
-      "version": "1.13.0",
-      "from": "pg-types@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz"
-    },
-    "pgpass": {
-      "version": "1.0.2",
-      "from": "pgpass@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz"
-    },
-    "pify": {
-      "version": "2.3.0",
-      "from": "pify@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"
-    },
-    "pinkie": {
-      "version": "2.0.4",
-      "from": "pinkie@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"
-    },
-    "pinkie-promise": {
-      "version": "2.0.1",
-      "from": "pinkie-promise@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz"
-    },
-    "postgres-array": {
-      "version": "1.0.2",
-      "from": "postgres-array@>=1.0.0 <1.1.0",
-      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.2.tgz"
-    },
-    "postgres-bytea": {
-      "version": "1.0.0",
-      "from": "postgres-bytea@>=1.0.0 <1.1.0",
-      "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz"
-    },
-    "postgres-date": {
-      "version": "1.0.3",
-      "from": "postgres-date@>=1.0.0 <1.1.0",
-      "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz"
-    },
-    "postgres-interval": {
-      "version": "1.1.1",
-      "from": "postgres-interval@>=1.1.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.1.tgz"
-    },
-    "process-nextick-args": {
-      "version": "2.0.0",
-      "from": "process-nextick-args@>=2.0.0 <2.1.0",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz"
-    },
-    "proxy-addr": {
-      "version": "1.0.10",
-      "from": "proxy-addr@>=1.0.10 <1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.10.tgz"
-    },
-    "punycode": {
-      "version": "1.4.1",
-      "from": "punycode@>=1.4.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz"
-    },
-    "qs": {
-      "version": "6.2.3",
-      "from": "qs@>=6.2.1 <6.3.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.2.3.tgz"
-    },
-    "queue-async": {
-      "version": "1.0.7",
-      "from": "queue-async@>=1.0.7 <1.1.0",
-      "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz"
-    },
-    "range-parser": {
-      "version": "1.0.3",
-      "from": "range-parser@>=1.0.3 <1.1.0",
-      "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
-    },
-    "raw-body": {
-      "version": "2.3.2",
-      "from": "raw-body@2.3.2",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz",
-      "dependencies": {
-        "depd": {
-          "version": "1.1.1",
-          "from": "depd@1.1.1",
-          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz"
-        },
-        "http-errors": {
-          "version": "1.6.2",
-          "from": "http-errors@1.6.2",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz"
-        },
-        "setprototypeof": {
-          "version": "1.0.3",
-          "from": "setprototypeof@1.0.3",
-          "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz"
-        }
-      }
-    },
-    "read-pkg": {
-      "version": "1.1.0",
-      "from": "read-pkg@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz"
-    },
-    "read-pkg-up": {
-      "version": "1.0.1",
-      "from": "read-pkg-up@>=1.0.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz"
-    },
-    "readable-stream": {
-      "version": "1.0.34",
-      "from": "readable-stream@>=1.0.2 <1.1.0",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz"
-    },
-    "redis": {
-      "version": "2.8.0",
-      "from": "redis@>=2.8.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz"
-    },
-    "redis-commands": {
-      "version": "1.3.5",
-      "from": "redis-commands@>=1.2.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz"
-    },
-    "redis-mpool": {
-      "version": "0.5.0",
-      "from": "redis-mpool@0.5.0",
-      "resolved": "https://registry.npmjs.org/redis-mpool/-/redis-mpool-0.5.0.tgz",
-      "dependencies": {
-        "generic-pool": {
-          "version": "2.1.1",
-          "from": "generic-pool@>=2.1.1 <2.2.0",
-          "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz"
-        }
-      }
-    },
-    "redis-parser": {
-      "version": "2.6.0",
-      "from": "redis-parser@>=2.6.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz"
-    },
-    "redlock": {
-      "version": "2.0.1",
-      "from": "redlock@2.0.1",
-      "resolved": "https://registry.npmjs.org/redlock/-/redlock-2.0.1.tgz"
-    },
-    "request": {
-      "version": "2.75.0",
-      "from": "request@>=2.75.0 <2.76.0",
-      "resolved": "https://registry.npmjs.org/request/-/request-2.75.0.tgz"
-    },
-    "require-directory": {
-      "version": "2.1.1",
-      "from": "require-directory@>=2.1.1 <3.0.0",
-      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz"
-    },
-    "require-main-filename": {
-      "version": "1.0.1",
-      "from": "require-main-filename@>=1.0.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz"
-    },
-    "rimraf": {
-      "version": "2.4.5",
-      "from": "rimraf@>=2.4.0 <2.5.0",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
-      "optional": true
-    },
-    "safe-buffer": {
-      "version": "5.1.1",
-      "from": "safe-buffer@5.1.1",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz"
-    },
-    "safe-json-stringify": {
-      "version": "1.1.0",
-      "from": "safe-json-stringify@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz",
-      "optional": true
-    },
-    "semver": {
-      "version": "4.3.2",
-      "from": "semver@4.3.2",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz"
-    },
-    "send": {
-      "version": "0.13.1",
-      "from": "send@0.13.1",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz",
-      "dependencies": {
-        "http-errors": {
-          "version": "1.3.1",
-          "from": "http-errors@>=1.3.1 <1.4.0",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz"
-        },
-        "ms": {
-          "version": "0.7.1",
-          "from": "ms@0.7.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-        },
-        "statuses": {
-          "version": "1.2.1",
-          "from": "statuses@>=1.2.1 <1.3.0",
-          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
-        }
-      }
-    },
-    "serve-static": {
-      "version": "1.10.3",
-      "from": "serve-static@>=1.10.2 <1.11.0",
-      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz",
-      "dependencies": {
-        "http-errors": {
-          "version": "1.3.1",
-          "from": "http-errors@>=1.3.1 <1.4.0",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz"
-        },
-        "ms": {
-          "version": "0.7.1",
-          "from": "ms@0.7.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-        },
-        "send": {
-          "version": "0.13.2",
-          "from": "send@0.13.2",
-          "resolved": "https://registry.npmjs.org/send/-/send-0.13.2.tgz"
-        },
-        "statuses": {
-          "version": "1.2.1",
-          "from": "statuses@>=1.2.1 <1.3.0",
-          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
-        }
-      }
-    },
-    "set-blocking": {
-      "version": "2.0.0",
-      "from": "set-blocking@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz"
-    },
-    "setprototypeof": {
-      "version": "1.1.0",
-      "from": "setprototypeof@1.1.0",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz"
-    },
-    "sntp": {
-      "version": "1.0.9",
-      "from": "sntp@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
-    },
-    "spdx-correct": {
-      "version": "3.0.0",
-      "from": "spdx-correct@>=3.0.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.0.tgz"
-    },
-    "spdx-exceptions": {
-      "version": "2.1.0",
-      "from": "spdx-exceptions@>=2.1.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.1.0.tgz"
-    },
-    "spdx-expression-parse": {
-      "version": "3.0.0",
-      "from": "spdx-expression-parse@>=3.0.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz"
-    },
-    "spdx-license-ids": {
-      "version": "3.0.0",
-      "from": "spdx-license-ids@>=3.0.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.0.tgz"
-    },
-    "split": {
-      "version": "1.0.1",
-      "from": "split@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz"
-    },
-    "sshpk": {
-      "version": "1.14.1",
-      "from": "sshpk@>=1.7.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz",
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "from": "assert-plus@>=1.0.0 <2.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
-        }
-      }
-    },
-    "statuses": {
-      "version": "1.5.0",
-      "from": "statuses@>=1.4.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz"
-    },
-    "step": {
-      "version": "0.0.6",
-      "from": "step@>=0.0.5 <0.1.0",
-      "resolved": "https://registry.npmjs.org/step/-/step-0.0.6.tgz"
-    },
-    "step-profiler": {
-      "version": "0.3.0",
-      "from": "step-profiler@>=0.3.0 <0.4.0",
-      "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz"
-    },
-    "streamsearch": {
-      "version": "0.1.2",
-      "from": "streamsearch@0.1.2",
-      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz"
-    },
-    "string_decoder": {
-      "version": "0.10.31",
-      "from": "string_decoder@>=0.10.0 <0.11.0",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
-    },
-    "string-width": {
-      "version": "1.0.2",
-      "from": "string-width@>=1.0.2 <2.0.0",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz"
-    },
-    "stringstream": {
-      "version": "0.0.5",
-      "from": "stringstream@>=0.0.4 <0.1.0",
-      "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz"
-    },
-    "strip-ansi": {
-      "version": "3.0.1",
-      "from": "strip-ansi@>=3.0.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz"
-    },
-    "strip-bom": {
-      "version": "2.0.0",
-      "from": "strip-bom@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz"
-    },
-    "supports-color": {
-      "version": "2.0.0",
-      "from": "supports-color@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz"
-    },
-    "through": {
-      "version": "2.3.8",
-      "from": "through@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz"
-    },
-    "topojson": {
-      "version": "0.0.8",
-      "from": "topojson@0.0.8",
-      "resolved": "https://registry.npmjs.org/topojson/-/topojson-0.0.8.tgz"
-    },
-    "tough-cookie": {
-      "version": "2.3.4",
-      "from": "tough-cookie@>=2.3.0 <2.4.0",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz"
-    },
-    "tunnel-agent": {
-      "version": "0.4.3",
-      "from": "tunnel-agent@>=0.4.1 <0.5.0",
-      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz"
-    },
-    "tweetnacl": {
-      "version": "0.14.5",
-      "from": "tweetnacl@>=0.14.0 <0.15.0",
-      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
-      "optional": true
-    },
-    "type-is": {
-      "version": "1.6.16",
-      "from": "type-is@>=1.6.15 <1.7.0",
-      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz"
-    },
-    "typedarray": {
-      "version": "0.0.6",
-      "from": "typedarray@>=0.0.6 <0.0.7",
-      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz"
-    },
-    "underscore": {
-      "version": "1.6.0",
-      "from": "underscore@>=1.6.0 <1.7.0",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
-    },
-    "unpipe": {
-      "version": "1.0.0",
-      "from": "unpipe@1.0.0",
-      "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
-    },
-    "util-deprecate": {
-      "version": "1.0.2",
-      "from": "util-deprecate@>=1.0.1 <1.1.0",
-      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
-    },
-    "utils-merge": {
-      "version": "1.0.0",
-      "from": "utils-merge@1.0.0",
-      "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
-    },
-    "validate-npm-package-license": {
-      "version": "3.0.3",
-      "from": "validate-npm-package-license@>=3.0.1 <4.0.0",
-      "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.3.tgz"
-    },
-    "vary": {
-      "version": "1.0.1",
-      "from": "vary@>=1.0.1 <1.1.0",
-      "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
-    },
-    "verror": {
-      "version": "1.10.0",
-      "from": "verror@1.10.0",
-      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "from": "assert-plus@>=1.0.0 <2.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
-        }
-      }
-    },
-    "which-module": {
-      "version": "1.0.0",
-      "from": "which-module@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz"
-    },
-    "window-size": {
-      "version": "0.2.0",
-      "from": "window-size@>=0.2.0 <0.3.0",
-      "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz"
-    },
-    "wordwrap": {
-      "version": "0.0.3",
-      "from": "wordwrap@>=0.0.2 <0.1.0",
-      "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz"
-    },
-    "wrap-ansi": {
-      "version": "2.1.0",
-      "from": "wrap-ansi@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz"
-    },
-    "wrappy": {
-      "version": "1.0.2",
-      "from": "wrappy@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"
-    },
-    "xtend": {
-      "version": "4.0.1",
-      "from": "xtend@>=4.0.0 <5.0.0",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz"
-    },
-    "y18n": {
-      "version": "3.2.1",
-      "from": "y18n@>=3.2.1 <4.0.0",
-      "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz"
-    },
-    "yargs": {
-      "version": "5.0.0",
-      "from": "yargs@>=5.0.0 <5.1.0",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-5.0.0.tgz"
-    },
-    "yargs-parser": {
-      "version": "3.2.0",
-      "from": "yargs-parser@>=3.2.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-3.2.0.tgz"
-    }
-  }
-}
diff --git a/package.json b/package.json
index 2cd03dea..d6b7a3f6 100644
--- a/package.json
+++ b/package.json
@@ -28,7 +28,6 @@
     "express": "~4.13.3",
     "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "~2.5.0",
-    "multer": "~1.3.0",
     "node-statsd": "~0.0.7",
     "node-uuid": "^1.4.7",
     "oauth-client": "0.3.0",

From fa4cbfac190a526ff87180c3f8a75b367343f856 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 18:49:59 +0200
Subject: [PATCH 043/133] dependencies

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index d6b7a3f6..90559328 100644
--- a/package.json
+++ b/package.json
@@ -18,9 +18,9 @@
   ],
   "dependencies": {
     "basic-auth": "^2.0.0",
-    "body-parser":  "~1.18.2",
     "bintrees": "1.0.1",
     "bunyan": "1.8.1",
+    "busboy": "^0.2.14",
     "cartodb-psql": "0.10.1",
     "cartodb-query-tables": "0.2.0",
     "cartodb-redis": "1.0.0",

From dddfc806af0230d0d36fef40336f8df3eef7dd0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 8 May 2018 18:55:15 +0200
Subject: [PATCH 044/133] shrinkwrap

---
 npm-shrinkwrap.json | 1184 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1184 insertions(+)
 create mode 100644 npm-shrinkwrap.json

diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
new file mode 100644
index 00000000..4386c33d
--- /dev/null
+++ b/npm-shrinkwrap.json
@@ -0,0 +1,1184 @@
+{
+  "name": "cartodb_sql_api",
+  "version": "2.0.1",
+  "dependencies": {
+    "accepts": {
+      "version": "1.2.13",
+      "from": "accepts@>=1.2.12 <1.3.0",
+      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz"
+    },
+    "ansi-regex": {
+      "version": "2.1.1",
+      "from": "ansi-regex@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz"
+    },
+    "ansi-styles": {
+      "version": "2.2.1",
+      "from": "ansi-styles@>=2.2.1 <3.0.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz"
+    },
+    "array-flatten": {
+      "version": "1.1.1",
+      "from": "array-flatten@1.1.1",
+      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
+    },
+    "asn1": {
+      "version": "0.2.3",
+      "from": "asn1@>=0.2.3 <0.3.0",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz"
+    },
+    "assert-plus": {
+      "version": "0.2.0",
+      "from": "assert-plus@>=0.2.0 <0.3.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz"
+    },
+    "async": {
+      "version": "0.2.10",
+      "from": "async@>=0.2.0 <0.3.0",
+      "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz"
+    },
+    "asynckit": {
+      "version": "0.4.0",
+      "from": "asynckit@>=0.4.0 <0.5.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz"
+    },
+    "aws-sign2": {
+      "version": "0.6.0",
+      "from": "aws-sign2@>=0.6.0 <0.7.0",
+      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
+    },
+    "aws4": {
+      "version": "1.7.0",
+      "from": "aws4@>=1.2.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz"
+    },
+    "balanced-match": {
+      "version": "1.0.0",
+      "from": "balanced-match@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz"
+    },
+    "basic-auth": {
+      "version": "2.0.0",
+      "from": "basic-auth@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.0.tgz"
+    },
+    "bcrypt-pbkdf": {
+      "version": "1.0.1",
+      "from": "bcrypt-pbkdf@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
+      "optional": true
+    },
+    "bindings": {
+      "version": "1.3.0",
+      "from": "bindings@>=1.2.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz"
+    },
+    "bintrees": {
+      "version": "1.0.1",
+      "from": "bintrees@1.0.1",
+      "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz"
+    },
+    "bl": {
+      "version": "1.1.2",
+      "from": "bl@>=1.1.2 <1.2.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz",
+      "dependencies": {
+        "isarray": {
+          "version": "1.0.0",
+          "from": "isarray@>=1.0.0 <1.1.0",
+          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
+        },
+        "readable-stream": {
+          "version": "2.0.6",
+          "from": "readable-stream@>=2.0.5 <2.1.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz"
+        }
+      }
+    },
+    "bluebird": {
+      "version": "3.5.1",
+      "from": "bluebird@>=3.3.3 <4.0.0",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz"
+    },
+    "boom": {
+      "version": "2.10.1",
+      "from": "boom@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "from": "brace-expansion@>=1.1.7 <2.0.0",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz"
+    },
+    "buffer-writer": {
+      "version": "1.0.1",
+      "from": "buffer-writer@1.0.1",
+      "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz"
+    },
+    "builtin-modules": {
+      "version": "1.1.1",
+      "from": "builtin-modules@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz"
+    },
+    "bunyan": {
+      "version": "1.8.1",
+      "from": "bunyan@1.8.1",
+      "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.1.tgz"
+    },
+    "busboy": {
+      "version": "0.2.14",
+      "from": "busboy@>=0.2.14 <0.3.0",
+      "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz"
+    },
+    "camelcase": {
+      "version": "3.0.0",
+      "from": "camelcase@>=3.0.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz"
+    },
+    "cartodb-psql": {
+      "version": "0.10.1",
+      "from": "cartodb-psql@0.10.1",
+      "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.10.1.tgz"
+    },
+    "cartodb-query-tables": {
+      "version": "0.2.0",
+      "from": "cartodb-query-tables@0.2.0",
+      "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz"
+    },
+    "cartodb-redis": {
+      "version": "1.0.0",
+      "from": "cartodb-redis@1.0.0",
+      "resolved": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-1.0.0.tgz"
+    },
+    "caseless": {
+      "version": "0.11.0",
+      "from": "caseless@>=0.11.0 <0.12.0",
+      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz"
+    },
+    "chalk": {
+      "version": "1.1.3",
+      "from": "chalk@>=1.1.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz"
+    },
+    "cliui": {
+      "version": "3.2.0",
+      "from": "cliui@>=3.2.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz"
+    },
+    "code-point-at": {
+      "version": "1.1.0",
+      "from": "code-point-at@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz"
+    },
+    "combined-stream": {
+      "version": "1.0.6",
+      "from": "combined-stream@>=1.0.5 <1.1.0",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz"
+    },
+    "commander": {
+      "version": "2.15.1",
+      "from": "commander@>=2.9.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz"
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "from": "concat-map@0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz"
+    },
+    "content-disposition": {
+      "version": "0.5.1",
+      "from": "content-disposition@0.5.1",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz"
+    },
+    "content-type": {
+      "version": "1.0.4",
+      "from": "content-type@>=1.0.1 <1.1.0",
+      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz"
+    },
+    "cookie": {
+      "version": "0.1.5",
+      "from": "cookie@0.1.5",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz"
+    },
+    "cookie-signature": {
+      "version": "1.0.6",
+      "from": "cookie-signature@1.0.6",
+      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "from": "core-util-is@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
+    },
+    "cryptiles": {
+      "version": "2.0.5",
+      "from": "cryptiles@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
+    },
+    "dashdash": {
+      "version": "1.14.1",
+      "from": "dashdash@>=1.12.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+      "dependencies": {
+        "assert-plus": {
+          "version": "1.0.0",
+          "from": "assert-plus@>=1.0.0 <2.0.0",
+          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+        }
+      }
+    },
+    "debug": {
+      "version": "2.2.0",
+      "from": "debug@2.2.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz"
+    },
+    "decamelize": {
+      "version": "1.2.0",
+      "from": "decamelize@>=1.1.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz"
+    },
+    "delayed-stream": {
+      "version": "1.0.0",
+      "from": "delayed-stream@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz"
+    },
+    "depd": {
+      "version": "1.1.2",
+      "from": "depd@>=1.1.0 <1.2.0",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz"
+    },
+    "destroy": {
+      "version": "1.0.4",
+      "from": "destroy@>=1.0.4 <1.1.0",
+      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz"
+    },
+    "dicer": {
+      "version": "0.2.5",
+      "from": "dicer@0.2.5",
+      "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz"
+    },
+    "dot": {
+      "version": "1.0.3",
+      "from": "dot@>=1.0.2 <1.1.0",
+      "resolved": "https://registry.npmjs.org/dot/-/dot-1.0.3.tgz"
+    },
+    "double-ended-queue": {
+      "version": "2.1.0-0",
+      "from": "double-ended-queue@>=2.1.0-0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz"
+    },
+    "dtrace-provider": {
+      "version": "0.6.0",
+      "from": "dtrace-provider@>=0.6.0 <0.7.0",
+      "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz",
+      "optional": true
+    },
+    "ecc-jsbn": {
+      "version": "0.1.1",
+      "from": "ecc-jsbn@>=0.1.1 <0.2.0",
+      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
+      "optional": true
+    },
+    "ee-first": {
+      "version": "1.1.1",
+      "from": "ee-first@1.1.1",
+      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+    },
+    "error-ex": {
+      "version": "1.3.1",
+      "from": "error-ex@>=1.2.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz"
+    },
+    "escape-html": {
+      "version": "1.0.3",
+      "from": "escape-html@>=1.0.3 <1.1.0",
+      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz"
+    },
+    "escape-string-regexp": {
+      "version": "1.0.5",
+      "from": "escape-string-regexp@>=1.0.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
+    },
+    "etag": {
+      "version": "1.7.0",
+      "from": "etag@>=1.7.0 <1.8.0",
+      "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
+    },
+    "express": {
+      "version": "4.13.4",
+      "from": "express@>=4.13.3 <4.14.0",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.13.4.tgz",
+      "dependencies": {
+        "qs": {
+          "version": "4.0.0",
+          "from": "qs@4.0.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
+        }
+      }
+    },
+    "extend": {
+      "version": "3.0.1",
+      "from": "extend@>=3.0.0 <3.1.0",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz"
+    },
+    "extsprintf": {
+      "version": "1.3.0",
+      "from": "extsprintf@1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz"
+    },
+    "finalhandler": {
+      "version": "0.4.1",
+      "from": "finalhandler@0.4.1",
+      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz"
+    },
+    "find-up": {
+      "version": "1.1.2",
+      "from": "find-up@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz"
+    },
+    "forever-agent": {
+      "version": "0.6.1",
+      "from": "forever-agent@>=0.6.1 <0.7.0",
+      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
+    },
+    "form-data": {
+      "version": "2.0.0",
+      "from": "form-data@>=2.0.0 <2.1.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.0.0.tgz"
+    },
+    "forwarded": {
+      "version": "0.1.2",
+      "from": "forwarded@>=0.1.0 <0.2.0",
+      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz"
+    },
+    "fresh": {
+      "version": "0.3.0",
+      "from": "fresh@0.3.0",
+      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
+    },
+    "generate-function": {
+      "version": "2.0.0",
+      "from": "generate-function@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz"
+    },
+    "generate-object-property": {
+      "version": "1.2.0",
+      "from": "generate-object-property@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz"
+    },
+    "generic-pool": {
+      "version": "2.4.3",
+      "from": "generic-pool@2.4.3",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.3.tgz"
+    },
+    "get-caller-file": {
+      "version": "1.0.2",
+      "from": "get-caller-file@>=1.0.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz"
+    },
+    "getpass": {
+      "version": "0.1.7",
+      "from": "getpass@>=0.1.1 <0.2.0",
+      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+      "dependencies": {
+        "assert-plus": {
+          "version": "1.0.0",
+          "from": "assert-plus@>=1.0.0 <2.0.0",
+          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+        }
+      }
+    },
+    "glob": {
+      "version": "6.0.4",
+      "from": "glob@>=6.0.1 <7.0.0",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
+      "optional": true
+    },
+    "graceful-fs": {
+      "version": "4.1.11",
+      "from": "graceful-fs@>=4.1.2 <5.0.0",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz"
+    },
+    "har-validator": {
+      "version": "2.0.6",
+      "from": "har-validator@>=2.0.6 <2.1.0",
+      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz"
+    },
+    "has-ansi": {
+      "version": "2.0.0",
+      "from": "has-ansi@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz"
+    },
+    "hawk": {
+      "version": "3.1.3",
+      "from": "hawk@>=3.1.3 <3.2.0",
+      "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz"
+    },
+    "hiredis": {
+      "version": "0.5.0",
+      "from": "hiredis@>=0.5.0 <0.6.0",
+      "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.5.0.tgz"
+    },
+    "hoek": {
+      "version": "2.16.3",
+      "from": "hoek@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
+    },
+    "hosted-git-info": {
+      "version": "2.6.0",
+      "from": "hosted-git-info@>=2.1.4 <3.0.0",
+      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.6.0.tgz"
+    },
+    "http-errors": {
+      "version": "1.3.1",
+      "from": "http-errors@>=1.3.1 <1.4.0",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz"
+    },
+    "http-signature": {
+      "version": "1.1.1",
+      "from": "http-signature@>=1.1.0 <1.2.0",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz"
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "from": "inflight@>=1.0.4 <2.0.0",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz"
+    },
+    "inherits": {
+      "version": "2.0.3",
+      "from": "inherits@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
+    },
+    "invert-kv": {
+      "version": "1.0.0",
+      "from": "invert-kv@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz"
+    },
+    "ipaddr.js": {
+      "version": "1.0.5",
+      "from": "ipaddr.js@1.0.5",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.5.tgz"
+    },
+    "is-arrayish": {
+      "version": "0.2.1",
+      "from": "is-arrayish@>=0.2.1 <0.3.0",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz"
+    },
+    "is-builtin-module": {
+      "version": "1.0.0",
+      "from": "is-builtin-module@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz"
+    },
+    "is-fullwidth-code-point": {
+      "version": "1.0.0",
+      "from": "is-fullwidth-code-point@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz"
+    },
+    "is-my-ip-valid": {
+      "version": "1.0.0",
+      "from": "is-my-ip-valid@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/is-my-ip-valid/-/is-my-ip-valid-1.0.0.tgz"
+    },
+    "is-my-json-valid": {
+      "version": "2.17.2",
+      "from": "is-my-json-valid@>=2.12.4 <3.0.0",
+      "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.17.2.tgz"
+    },
+    "is-property": {
+      "version": "1.0.2",
+      "from": "is-property@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz"
+    },
+    "is-typedarray": {
+      "version": "1.0.0",
+      "from": "is-typedarray@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz"
+    },
+    "is-utf8": {
+      "version": "0.2.1",
+      "from": "is-utf8@>=0.2.0 <0.3.0",
+      "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz"
+    },
+    "isarray": {
+      "version": "0.0.1",
+      "from": "isarray@0.0.1",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
+    },
+    "isstream": {
+      "version": "0.1.2",
+      "from": "isstream@>=0.1.2 <0.2.0",
+      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
+    },
+    "js-string-escape": {
+      "version": "1.0.1",
+      "from": "js-string-escape@1.0.1",
+      "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz"
+    },
+    "jsbn": {
+      "version": "0.1.1",
+      "from": "jsbn@>=0.1.0 <0.2.0",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+      "optional": true
+    },
+    "json-schema": {
+      "version": "0.2.3",
+      "from": "json-schema@0.2.3",
+      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz"
+    },
+    "json-stringify-safe": {
+      "version": "5.0.1",
+      "from": "json-stringify-safe@>=5.0.1 <5.1.0",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
+    },
+    "jsonpointer": {
+      "version": "4.0.1",
+      "from": "jsonpointer@>=4.0.0 <5.0.0",
+      "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.1.tgz"
+    },
+    "jsprim": {
+      "version": "1.4.1",
+      "from": "jsprim@>=1.2.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
+      "dependencies": {
+        "assert-plus": {
+          "version": "1.0.0",
+          "from": "assert-plus@1.0.0",
+          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+        }
+      }
+    },
+    "lcid": {
+      "version": "1.0.0",
+      "from": "lcid@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz"
+    },
+    "load-json-file": {
+      "version": "1.1.0",
+      "from": "load-json-file@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz"
+    },
+    "lodash.assign": {
+      "version": "4.2.0",
+      "from": "lodash.assign@>=4.2.0 <5.0.0",
+      "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz"
+    },
+    "log4js": {
+      "version": "0.6.25",
+      "from": "cartodb/log4js-node#cdb",
+      "resolved": "git://github.com/cartodb/log4js-node.git#145d5f91e35e7fb14a6278cbf7a711ced6603727",
+      "dependencies": {
+        "readable-stream": {
+          "version": "1.0.34",
+          "from": "readable-stream@>=1.0.2 <1.1.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz"
+        },
+        "semver": {
+          "version": "4.3.6",
+          "from": "semver@>=4.3.3 <4.4.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz"
+        },
+        "underscore": {
+          "version": "1.8.2",
+          "from": "underscore@1.8.2",
+          "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.2.tgz"
+        }
+      }
+    },
+    "lru-cache": {
+      "version": "2.5.2",
+      "from": "lru-cache@>=2.5.0 <2.6.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.2.tgz"
+    },
+    "media-typer": {
+      "version": "0.3.0",
+      "from": "media-typer@0.3.0",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+    },
+    "merge-descriptors": {
+      "version": "1.0.1",
+      "from": "merge-descriptors@1.0.1",
+      "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz"
+    },
+    "methods": {
+      "version": "1.1.2",
+      "from": "methods@>=1.1.2 <1.2.0",
+      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
+    },
+    "mime": {
+      "version": "1.3.4",
+      "from": "mime@1.3.4",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
+    },
+    "mime-db": {
+      "version": "1.33.0",
+      "from": "mime-db@>=1.33.0 <1.34.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz"
+    },
+    "mime-types": {
+      "version": "2.1.18",
+      "from": "mime-types@>=2.1.6 <2.2.0",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz"
+    },
+    "minimatch": {
+      "version": "3.0.4",
+      "from": "minimatch@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz"
+    },
+    "minimist": {
+      "version": "0.0.8",
+      "from": "minimist@0.0.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz"
+    },
+    "mkdirp": {
+      "version": "0.5.1",
+      "from": "mkdirp@>=0.5.1 <0.6.0",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz"
+    },
+    "moment": {
+      "version": "2.22.1",
+      "from": "moment@>=2.10.6 <3.0.0",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz",
+      "optional": true
+    },
+    "ms": {
+      "version": "0.7.1",
+      "from": "ms@0.7.1",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+    },
+    "mv": {
+      "version": "2.1.1",
+      "from": "mv@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
+      "optional": true
+    },
+    "nan": {
+      "version": "2.10.0",
+      "from": "nan@>=2.0.8 <3.0.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz"
+    },
+    "ncp": {
+      "version": "2.0.0",
+      "from": "ncp@>=2.0.0 <2.1.0",
+      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
+      "optional": true
+    },
+    "negotiator": {
+      "version": "0.5.3",
+      "from": "negotiator@0.5.3",
+      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
+    },
+    "node-statsd": {
+      "version": "0.0.7",
+      "from": "node-statsd@>=0.0.7 <0.1.0",
+      "resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.0.7.tgz"
+    },
+    "node-uuid": {
+      "version": "1.4.8",
+      "from": "node-uuid@>=1.4.7 <2.0.0",
+      "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz"
+    },
+    "normalize-package-data": {
+      "version": "2.4.0",
+      "from": "normalize-package-data@>=2.3.2 <3.0.0",
+      "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz"
+    },
+    "number-is-nan": {
+      "version": "1.0.1",
+      "from": "number-is-nan@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz"
+    },
+    "oauth-client": {
+      "version": "0.3.0",
+      "from": "oauth-client@0.3.0",
+      "resolved": "https://registry.npmjs.org/oauth-client/-/oauth-client-0.3.0.tgz",
+      "dependencies": {
+        "node-uuid": {
+          "version": "1.1.0",
+          "from": "node-uuid@1.1.0",
+          "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.1.0.tgz"
+        }
+      }
+    },
+    "oauth-sign": {
+      "version": "0.8.2",
+      "from": "oauth-sign@>=0.8.1 <0.9.0",
+      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz"
+    },
+    "object-assign": {
+      "version": "4.1.0",
+      "from": "object-assign@4.1.0",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.0.tgz"
+    },
+    "on-finished": {
+      "version": "2.3.0",
+      "from": "on-finished@>=2.3.0 <2.4.0",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz"
+    },
+    "once": {
+      "version": "1.4.0",
+      "from": "once@>=1.3.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz"
+    },
+    "optimist": {
+      "version": "0.3.5",
+      "from": "optimist@0.3.5",
+      "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.5.tgz"
+    },
+    "os-locale": {
+      "version": "1.4.0",
+      "from": "os-locale@>=1.4.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz"
+    },
+    "packet-reader": {
+      "version": "0.2.0",
+      "from": "packet-reader@0.2.0",
+      "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.2.0.tgz"
+    },
+    "parse-json": {
+      "version": "2.2.0",
+      "from": "parse-json@>=2.2.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz"
+    },
+    "parseurl": {
+      "version": "1.3.2",
+      "from": "parseurl@>=1.3.1 <1.4.0",
+      "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz"
+    },
+    "path-exists": {
+      "version": "2.1.0",
+      "from": "path-exists@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz"
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "from": "path-is-absolute@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz"
+    },
+    "path-to-regexp": {
+      "version": "0.1.7",
+      "from": "path-to-regexp@0.1.7",
+      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
+    },
+    "path-type": {
+      "version": "1.1.0",
+      "from": "path-type@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz"
+    },
+    "pg": {
+      "version": "6.1.6",
+      "from": "cartodb/node-postgres#6.1.6-cdb1",
+      "resolved": "git://github.com/cartodb/node-postgres.git#3eef52dd1e655f658a4ee8ac5697688b3ecfed44"
+    },
+    "pg-connection-string": {
+      "version": "0.1.3",
+      "from": "pg-connection-string@0.1.3",
+      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz"
+    },
+    "pg-copy-streams": {
+      "version": "1.2.0",
+      "from": "pg-copy-streams@>=1.2.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-1.2.0.tgz"
+    },
+    "pg-int8": {
+      "version": "1.0.1",
+      "from": "pg-int8@1.0.1",
+      "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz"
+    },
+    "pg-pool": {
+      "version": "1.8.0",
+      "from": "pg-pool@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-1.8.0.tgz"
+    },
+    "pg-types": {
+      "version": "1.13.0",
+      "from": "pg-types@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz"
+    },
+    "pgpass": {
+      "version": "1.0.2",
+      "from": "pgpass@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz"
+    },
+    "pify": {
+      "version": "2.3.0",
+      "from": "pify@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"
+    },
+    "pinkie": {
+      "version": "2.0.4",
+      "from": "pinkie@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"
+    },
+    "pinkie-promise": {
+      "version": "2.0.1",
+      "from": "pinkie-promise@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz"
+    },
+    "postgres-array": {
+      "version": "1.0.2",
+      "from": "postgres-array@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.2.tgz"
+    },
+    "postgres-bytea": {
+      "version": "1.0.0",
+      "from": "postgres-bytea@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz"
+    },
+    "postgres-date": {
+      "version": "1.0.3",
+      "from": "postgres-date@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz"
+    },
+    "postgres-interval": {
+      "version": "1.1.1",
+      "from": "postgres-interval@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.1.tgz"
+    },
+    "process-nextick-args": {
+      "version": "1.0.7",
+      "from": "process-nextick-args@>=1.0.6 <1.1.0",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz"
+    },
+    "proxy-addr": {
+      "version": "1.0.10",
+      "from": "proxy-addr@>=1.0.10 <1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.10.tgz"
+    },
+    "punycode": {
+      "version": "1.4.1",
+      "from": "punycode@>=1.4.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz"
+    },
+    "qs": {
+      "version": "6.2.3",
+      "from": "qs@>=6.2.1 <6.3.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.2.3.tgz"
+    },
+    "queue-async": {
+      "version": "1.0.7",
+      "from": "queue-async@>=1.0.7 <1.1.0",
+      "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz"
+    },
+    "range-parser": {
+      "version": "1.0.3",
+      "from": "range-parser@>=1.0.3 <1.1.0",
+      "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
+    },
+    "read-pkg": {
+      "version": "1.1.0",
+      "from": "read-pkg@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz"
+    },
+    "read-pkg-up": {
+      "version": "1.0.1",
+      "from": "read-pkg-up@>=1.0.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz"
+    },
+    "readable-stream": {
+      "version": "1.1.14",
+      "from": "readable-stream@>=1.1.0 <1.2.0",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz"
+    },
+    "redis": {
+      "version": "2.8.0",
+      "from": "redis@>=2.8.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz"
+    },
+    "redis-commands": {
+      "version": "1.3.5",
+      "from": "redis-commands@>=1.2.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz"
+    },
+    "redis-mpool": {
+      "version": "0.5.0",
+      "from": "redis-mpool@0.5.0",
+      "resolved": "https://registry.npmjs.org/redis-mpool/-/redis-mpool-0.5.0.tgz",
+      "dependencies": {
+        "generic-pool": {
+          "version": "2.1.1",
+          "from": "generic-pool@>=2.1.1 <2.2.0",
+          "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz"
+        }
+      }
+    },
+    "redis-parser": {
+      "version": "2.6.0",
+      "from": "redis-parser@>=2.6.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz"
+    },
+    "redlock": {
+      "version": "2.0.1",
+      "from": "redlock@2.0.1",
+      "resolved": "https://registry.npmjs.org/redlock/-/redlock-2.0.1.tgz"
+    },
+    "request": {
+      "version": "2.75.0",
+      "from": "request@>=2.75.0 <2.76.0",
+      "resolved": "https://registry.npmjs.org/request/-/request-2.75.0.tgz"
+    },
+    "require-directory": {
+      "version": "2.1.1",
+      "from": "require-directory@>=2.1.1 <3.0.0",
+      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz"
+    },
+    "require-main-filename": {
+      "version": "1.0.1",
+      "from": "require-main-filename@>=1.0.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz"
+    },
+    "rimraf": {
+      "version": "2.4.5",
+      "from": "rimraf@>=2.4.0 <2.5.0",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
+      "optional": true
+    },
+    "safe-buffer": {
+      "version": "5.1.1",
+      "from": "safe-buffer@5.1.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz"
+    },
+    "safe-json-stringify": {
+      "version": "1.1.0",
+      "from": "safe-json-stringify@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz",
+      "optional": true
+    },
+    "semver": {
+      "version": "4.3.2",
+      "from": "semver@4.3.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz"
+    },
+    "send": {
+      "version": "0.13.1",
+      "from": "send@0.13.1",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz"
+    },
+    "serve-static": {
+      "version": "1.10.3",
+      "from": "serve-static@>=1.10.2 <1.11.0",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz",
+      "dependencies": {
+        "send": {
+          "version": "0.13.2",
+          "from": "send@0.13.2",
+          "resolved": "https://registry.npmjs.org/send/-/send-0.13.2.tgz"
+        }
+      }
+    },
+    "set-blocking": {
+      "version": "2.0.0",
+      "from": "set-blocking@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz"
+    },
+    "sntp": {
+      "version": "1.0.9",
+      "from": "sntp@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
+    },
+    "spdx-correct": {
+      "version": "3.0.0",
+      "from": "spdx-correct@>=3.0.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.0.tgz"
+    },
+    "spdx-exceptions": {
+      "version": "2.1.0",
+      "from": "spdx-exceptions@>=2.1.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.1.0.tgz"
+    },
+    "spdx-expression-parse": {
+      "version": "3.0.0",
+      "from": "spdx-expression-parse@>=3.0.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz"
+    },
+    "spdx-license-ids": {
+      "version": "3.0.0",
+      "from": "spdx-license-ids@>=3.0.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.0.tgz"
+    },
+    "split": {
+      "version": "1.0.1",
+      "from": "split@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz"
+    },
+    "sshpk": {
+      "version": "1.14.1",
+      "from": "sshpk@>=1.7.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz",
+      "dependencies": {
+        "assert-plus": {
+          "version": "1.0.0",
+          "from": "assert-plus@>=1.0.0 <2.0.0",
+          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+        }
+      }
+    },
+    "statuses": {
+      "version": "1.2.1",
+      "from": "statuses@>=1.2.1 <1.3.0",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+    },
+    "step": {
+      "version": "0.0.6",
+      "from": "step@>=0.0.5 <0.1.0",
+      "resolved": "https://registry.npmjs.org/step/-/step-0.0.6.tgz"
+    },
+    "step-profiler": {
+      "version": "0.3.0",
+      "from": "step-profiler@>=0.3.0 <0.4.0",
+      "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz"
+    },
+    "streamsearch": {
+      "version": "0.1.2",
+      "from": "streamsearch@0.1.2",
+      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz"
+    },
+    "string_decoder": {
+      "version": "0.10.31",
+      "from": "string_decoder@>=0.10.0 <0.11.0",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+    },
+    "string-width": {
+      "version": "1.0.2",
+      "from": "string-width@>=1.0.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz"
+    },
+    "stringstream": {
+      "version": "0.0.5",
+      "from": "stringstream@>=0.0.4 <0.1.0",
+      "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz"
+    },
+    "strip-ansi": {
+      "version": "3.0.1",
+      "from": "strip-ansi@>=3.0.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz"
+    },
+    "strip-bom": {
+      "version": "2.0.0",
+      "from": "strip-bom@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz"
+    },
+    "supports-color": {
+      "version": "2.0.0",
+      "from": "supports-color@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz"
+    },
+    "through": {
+      "version": "2.3.8",
+      "from": "through@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz"
+    },
+    "topojson": {
+      "version": "0.0.8",
+      "from": "topojson@0.0.8",
+      "resolved": "https://registry.npmjs.org/topojson/-/topojson-0.0.8.tgz"
+    },
+    "tough-cookie": {
+      "version": "2.3.4",
+      "from": "tough-cookie@>=2.3.0 <2.4.0",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz"
+    },
+    "tunnel-agent": {
+      "version": "0.4.3",
+      "from": "tunnel-agent@>=0.4.1 <0.5.0",
+      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz"
+    },
+    "tweetnacl": {
+      "version": "0.14.5",
+      "from": "tweetnacl@>=0.14.0 <0.15.0",
+      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
+      "optional": true
+    },
+    "type-is": {
+      "version": "1.6.16",
+      "from": "type-is@>=1.6.6 <1.7.0",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz"
+    },
+    "underscore": {
+      "version": "1.6.0",
+      "from": "underscore@>=1.6.0 <1.7.0",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
+    },
+    "unpipe": {
+      "version": "1.0.0",
+      "from": "unpipe@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
+    },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "from": "util-deprecate@>=1.0.1 <1.1.0",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
+    },
+    "utils-merge": {
+      "version": "1.0.0",
+      "from": "utils-merge@1.0.0",
+      "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
+    },
+    "validate-npm-package-license": {
+      "version": "3.0.3",
+      "from": "validate-npm-package-license@>=3.0.1 <4.0.0",
+      "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.3.tgz"
+    },
+    "vary": {
+      "version": "1.0.1",
+      "from": "vary@>=1.0.1 <1.1.0",
+      "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
+    },
+    "verror": {
+      "version": "1.10.0",
+      "from": "verror@1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "dependencies": {
+        "assert-plus": {
+          "version": "1.0.0",
+          "from": "assert-plus@>=1.0.0 <2.0.0",
+          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+        }
+      }
+    },
+    "which-module": {
+      "version": "1.0.0",
+      "from": "which-module@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz"
+    },
+    "window-size": {
+      "version": "0.2.0",
+      "from": "window-size@>=0.2.0 <0.3.0",
+      "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz"
+    },
+    "wordwrap": {
+      "version": "0.0.3",
+      "from": "wordwrap@>=0.0.2 <0.1.0",
+      "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz"
+    },
+    "wrap-ansi": {
+      "version": "2.1.0",
+      "from": "wrap-ansi@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz"
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "from": "wrappy@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"
+    },
+    "xtend": {
+      "version": "4.0.1",
+      "from": "xtend@>=4.0.0 <5.0.0",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz"
+    },
+    "y18n": {
+      "version": "3.2.1",
+      "from": "y18n@>=3.2.1 <4.0.0",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz"
+    },
+    "yargs": {
+      "version": "5.0.0",
+      "from": "yargs@>=5.0.0 <5.1.0",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-5.0.0.tgz"
+    },
+    "yargs-parser": {
+      "version": "3.2.0",
+      "from": "yargs-parser@>=3.2.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-3.2.0.tgz"
+    }
+  }
+}

From f0021264d2171ef339f58326e5f2ba2e4e7803e4 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 12:11:55 -0700
Subject: [PATCH 045/133] Change from form-multipart to POST with chunked
 upload

---
 app/controllers/copy_controller.js | 92 +++++++++++-------------------
 doc/copy_queries.md                | 23 ++++----
 test/acceptance/copy-endpoints.js  | 52 ++++++-----------
 3 files changed, 60 insertions(+), 107 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 474edb9b..73511622 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -8,7 +8,7 @@ const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
 const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
-const Busboy = require('busboy');
+// const Busboy = require('busboy');
 
 const PSQL = require('cartodb-psql');
 const copyTo = require('pg-copy-streams').to;
@@ -105,74 +105,48 @@ CopyController.prototype.responseCopyTo = function (req, res) {
 
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
     let sql = req.query.sql;
-    let files = 0;
+        
+    // curl -vv -X POST --data-binary @test.csv "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom?api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961&sql=copy+foo+(i,t)+from+stdin+with+(format+csv,header+false)"
 
-    const busboy = new Busboy({ 
-        headers: req.headers,
-        limits: {
-            fieldSize: Infinity,
-            files: 1
-        }
-    });
+    // curl -vv -X POST --data-binary @test.csv "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom?api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961&sql=copy+foo+(i,t)+from+stdin"
 
-    busboy.on('field', function (fieldname, val) {
-        if (fieldname === 'sql') {
-            sql = val;
 
-            // Only accept SQL that starts with 'COPY'
-            if (!sql.toUpperCase().startsWith("COPY ")) {
-                return next(new Error("SQL must start with COPY"));
+    // curl -X POST --data-binary @test.csv -G --data-urlencode "sql=COPY FOO (i, t) FROM stdin" --data-urlencode "api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961" "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom"
+
+    // curl -X POST -G --data-urlencode "sql=COPY FOO (i, t) FROM stdin" --data-urlencode "api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961" "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom"
+
+    
+    try {
+        const start_time = Date.now();
+
+        // Connect and run the COPY
+        const pg = new PSQL(res.locals.userDbParams);
+        pg.connect(function (err, client) {
+            if (err) {
+                return next(err);
             }
-        }
-    });
 
-    busboy.on('file', function (fieldname, file) {
-        files++;
+            let copyFromStream = copyFrom(sql);
+            const pgstream = client.query(copyFromStream);
+            pgstream.on('error', next);
+            pgstream.on('end', function () {
+                const end_time = Date.now();
+                res.body = {
+                    time: (end_time - start_time) / 1000,
+                    total_rows: copyFromStream.rowCount
+                };
 
-        if (!sql) {
-            return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
-        }
-
-        try {
-            const start_time = Date.now();
-
-            // Connect and run the COPY
-            const pg = new PSQL(res.locals.userDbParams);
-            pg.connect(function (err, client) {
-                if (err) {
-                    return next(err);
-                }
-
-                let copyFromStream = copyFrom(sql);
-                const pgstream = client.query(copyFromStream);
-                pgstream.on('error', next);
-                pgstream.on('end', function () {
-                    const end_time = Date.now();
-                    res.body = {
-                        time: (end_time - start_time) / 1000,
-                        total_rows: copyFromStream.rowCount
-                    };
-
-                    return next();
-                });
-
-                file.pipe(pgstream);
+                return next();
             });
 
-        } catch (err) {
-            next(err);
-        }
-    });
+            // req is a readable stream (cool!)
+            req.pipe(pgstream);
+        });
 
-    busboy.on('finish', () => {
-        if (files !== 1) {
-            return next(new Error("The file is missing"));
-        }
-    });
+    } catch (err) {
+        next(err);
+    }
 
-    busboy.on('error', next);
-
-    req.pipe(busboy);
 };
 
 CopyController.prototype.responseCopyFrom = function (req, res, next) {
diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 81c9867b..9b5cab08 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -4,8 +4,8 @@ Copy queries allow you to use the [PostgreSQL copy command](https://www.postgres
 
 The support for copy is split across two API end points:
 
-* `http://{username}.carto.com/api/v2/copyfrom` for uploading data to CARTO
-* `http://{username}.carto.com/api/v2/copyto` for exporting data out of CARTO
+* `http://{username}.carto.com/api/v2/sql/copyfrom` for uploading data to CARTO
+* `http://{username}.carto.com/api/v2/sql/copyto` for exporting data out of CARTO
 
 ## Copy From
 
@@ -68,12 +68,10 @@ The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from
 
 Assuming that you have already created the table, and that the CSV file is named "upload_example.csv":
 
-    curl \
-      --form sql="COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)" \
-      --form file=@upload_example.csv \
-      http://{username}.carto.com/api/v2/copyfrom?api_key={api_key}
-
-**Important:** When supplying the "sql" parameter as a form field, it must come **before** the "file" parameter, or the upload will fail. Alternatively, you can supply the "sql" parameter on the URL line.
+    curl --X POST \
+	    --data-binary @upload_example.csv \
+		"http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
+		
 
 ### Python Example
 
@@ -86,9 +84,10 @@ The [Requests](http://docs.python-requests.org/en/master/user/quickstart/) libra
     upload_file = 'upload_example.csv'
     sql = "COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)"
 
-    url = "http://%s.carto.com/api/v2/copyfrom" % username    
+    url = "http://%s.carto.com/api/v2/sql/copyfrom" % username    
     with open(upload_file, 'rb') as f:
-        r = requests.post(url, params={'api_key':api_key, 'sql':sql}, files={'file': f})
+        r = requests.post(url, params={'api_key':api_key, 'sql':sql}, data=f)
+		
         if r.status_code != 200:
             print r.text
         else:
@@ -146,7 +145,7 @@ The SQL needs to be URL-encoded before being embedded in the CURL command, so th
 
     curl \
         --output upload_example_dl.csv \
-        "http://{username}.carto.com/api/v2/copyto?sql=COPY+upload_example+(the_geom,name,age)+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
+        "http://{username}.carto.com/api/v2/sql/copyto?sql=COPY+upload_example+(the_geom,name,age)+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
 
 ### Python Example
 
@@ -161,7 +160,7 @@ The Python to "copy to" is very simple, because the HTTP call is a simple get. T
     sql = "COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)"
 
     # request the download, specifying desired file name
-    url = "http://%s.carto.com/api/v2/copyto" % username    
+    url = "http://%s.carto.com/api/v2/sql/copyto" % username    
     r = requests.get(url, params={'api_key':api_key, 'sql':sql, 'filename':download_file}, stream=True)
     r.raise_for_status()
 
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 8c84c1ec..0e428a1d 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -8,29 +8,10 @@ const assert = require('../support/assert');
 describe('copy-endpoints', function() {
     it('should works with copyfrom endpoint', function(done){
         assert.response(server, {
-            url: "/api/v1/sql/copyfrom",
-            formData: {
-                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
-                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
-            },
-            headers: {host: 'vizzuality.cartodb.com'},
-            method: 'POST'
-        },{}, function(err, res) {
-            assert.ifError(err);
-            const response = JSON.parse(res.body);
-            assert.equal(!!response.time, true);
-            assert.strictEqual(response.total_rows, 6);
-            done();
-        });
-    });
-    
-    it('should works with copyfrom endpoint and SQL in querystring', function(done){
-        const sql = "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)";
-        assert.response(server, {
-            url: `/api/v1/sql/copyfrom?sql=${sql}`,
-            formData: {
-                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
-            },
+            url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+            data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
         },{}, function(err, res) {
@@ -44,11 +25,11 @@ describe('copy-endpoints', function() {
 
     it('should fail with copyfrom endpoint and unexisting table', function(done){
         assert.response(server, {
-            url: "/api/v1/sql/copyfrom",
-            formData: {
-                sql: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
-                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
-            },
+            url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                sql: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+            
+            data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
         },{}, function(err, res) {
@@ -65,10 +46,9 @@ describe('copy-endpoints', function() {
 
     it('should fail with copyfrom endpoint and without csv', function(done){
         assert.response(server, {
-            url: "/api/v1/sql/copyfrom",
-            formData: {
-                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
-            },
+            url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
         },{}, function(err, res) {
@@ -85,10 +65,10 @@ describe('copy-endpoints', function() {
 
     it('should fail with copyfrom endpoint and without sql', function(done){
         assert.response(server, {
-            url: "/api/v1/sql/copyfrom",
-            formData: {
-                file: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
-            },
+            url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+            data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),            
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
         },{}, function(err, res) {

From 8f211f905ca5b83cae1f131a508cd9008f71b4e9 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 12:21:33 -0700
Subject: [PATCH 046/133] Remove wrong examples

---
 app/controllers/copy_controller.js | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 73511622..a11f53d3 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -8,7 +8,6 @@ const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
 const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
-// const Busboy = require('busboy');
 
 const PSQL = require('cartodb-psql');
 const copyTo = require('pg-copy-streams').to;
@@ -109,12 +108,6 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
     // curl -vv -X POST --data-binary @test.csv "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom?api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961&sql=copy+foo+(i,t)+from+stdin+with+(format+csv,header+false)"
 
     // curl -vv -X POST --data-binary @test.csv "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom?api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961&sql=copy+foo+(i,t)+from+stdin"
-
-
-    // curl -X POST --data-binary @test.csv -G --data-urlencode "sql=COPY FOO (i, t) FROM stdin" --data-urlencode "api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961" "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom"
-
-    // curl -X POST -G --data-urlencode "sql=COPY FOO (i, t) FROM stdin" --data-urlencode "api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961" "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom"
-
     
     try {
         const start_time = Date.now();

From d2ea840defd20934a8c9d780353fbfb181666179 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 12:24:12 -0700
Subject: [PATCH 047/133] Fix curl example for copyfrom

---
 doc/copy_queries.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 9b5cab08..7be07b32 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -68,7 +68,7 @@ The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from
 
 Assuming that you have already created the table, and that the CSV file is named "upload_example.csv":
 
-    curl --X POST \
+    curl -X POST \
 	    --data-binary @upload_example.csv \
 		"http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
 		

From c676b2f52052c2ea414a8e08179b6c61af036221 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 12:26:16 -0700
Subject: [PATCH 048/133] Ensure python example works with POST chunked

---
 doc/copy_queries.md | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 7be07b32..cf88a65b 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -104,13 +104,7 @@ A successful upload will return with status code 200, and a small JSON with info
 
 A failed upload will return with status code 400 and a larger JSON with the PostgreSQL error string, and a stack trace from the SQL API.
 
-    {"error":["Unexpected field"],
-     "stack":"Error: Unexpected field
-        at makeError (/repos/CartoDB-SQL-API/node_modules/multer/lib/make-error.js:12:13)
-        at wrappedFileFilter (/repos/CartoDB-SQL-API/node_modules/multer/index.js:39:19)
-        ...
-        at emitMany (events.js:127:13)
-        at SBMH.emit (events.js:201:7)"}
+    {"error":["Unexpected field"]}
     
 ## Copy To
 

From 58166eeda463ddcdaf47e9c9f81141208558d67d Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 12:37:19 -0700
Subject: [PATCH 049/133] Add compressed upload example

---
 doc/copy_queries.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index cf88a65b..6815622a 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -70,8 +70,17 @@ Assuming that you have already created the table, and that the CSV file is named
 
     curl -X POST \
 	    --data-binary @upload_example.csv \
+		-H "Content-Type: application/octet-stream" \
 		"http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
-		
+
+To upload a larger file, using compression for a faster transfer, first compress the file, and then upload it with the content encoding set:
+
+    curl -X POST  \
+		-H "Content-Encoding: gzip" \
+		-H "Content-Type: application/octet-stream"   \
+		--data-binary @upload_example.csv.gz \
+		"http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
+
 
 ### Python Example
 
@@ -139,6 +148,7 @@ The SQL needs to be URL-encoded before being embedded in the CURL command, so th
 
     curl \
         --output upload_example_dl.csv \
+		--compressed \
         "http://{username}.carto.com/api/v2/sql/copyto?sql=COPY+upload_example+(the_geom,name,age)+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
 
 ### Python Example

From 87c70066f508a4365fceda8f5086eebc15ecbeec Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 12:48:36 -0700
Subject: [PATCH 050/133] Add stream option to POST example

---
 doc/copy_queries.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 6815622a..227c8bae 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -95,7 +95,7 @@ The [Requests](http://docs.python-requests.org/en/master/user/quickstart/) libra
 
     url = "http://%s.carto.com/api/v2/sql/copyfrom" % username    
     with open(upload_file, 'rb') as f:
-        r = requests.post(url, params={'api_key':api_key, 'sql':sql}, data=f)
+        r = requests.post(url, params={'api_key':api_key, 'sql':sql}, data=f, stream=True)
 		
         if r.status_code != 200:
             print r.text

From 031d7d794b5af27b1192eff1b267563d4919f344 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 12:52:57 -0700
Subject: [PATCH 051/133] Add chunked header to CURL examples

---
 doc/copy_queries.md | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 227c8bae..bc24ccfe 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -69,17 +69,19 @@ The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from
 Assuming that you have already created the table, and that the CSV file is named "upload_example.csv":
 
     curl -X POST \
-	    --data-binary @upload_example.csv \
-		-H "Content-Type: application/octet-stream" \
-		"http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
+        --data-binary @upload_example.csv \
+        -H "Transfer-Encoding: chunked" \
+        -H "Content-Type: application/octet-stream" \
+        "http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
 
 To upload a larger file, using compression for a faster transfer, first compress the file, and then upload it with the content encoding set:
 
     curl -X POST  \
-		-H "Content-Encoding: gzip" \
-		-H "Content-Type: application/octet-stream"   \
-		--data-binary @upload_example.csv.gz \
-		"http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
+        -H "Content-Encoding: gzip" \
+        -H "Transfer-Encoding: chunked" \
+        -H "Content-Type: application/octet-stream"   \
+        --data-binary @upload_example.csv.gz \
+        "http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
 
 
 ### Python Example
@@ -96,7 +98,7 @@ The [Requests](http://docs.python-requests.org/en/master/user/quickstart/) libra
     url = "http://%s.carto.com/api/v2/sql/copyfrom" % username    
     with open(upload_file, 'rb') as f:
         r = requests.post(url, params={'api_key':api_key, 'sql':sql}, data=f, stream=True)
-		
+
         if r.status_code != 200:
             print r.text
         else:

From 1d3baaa05db5b1f01148c2b7235c9d6466138e3a Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Thu, 10 May 2018 13:48:10 -0700
Subject: [PATCH 052/133] Remove gzip post example, as it doesn't actually
 work. We could manually support content-encoding: gzip in express, but we'd
 have to hack body parsing a little, but it would make uploads potentially
 nicer/faster

---
 doc/copy_queries.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index bc24ccfe..fe0c2366 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -79,7 +79,7 @@ To upload a larger file, using compression for a faster transfer, first compress
     curl -X POST  \
         -H "Content-Encoding: gzip" \
         -H "Transfer-Encoding: chunked" \
-        -H "Content-Type: application/octet-stream"   \
+        -H "Content-Type: application/octet-stream" \
         --data-binary @upload_example.csv.gz \
         "http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
 

From 85528459b85fcb5371603427e09ee4bcb34bc651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 11 May 2018 10:38:04 +0200
Subject: [PATCH 053/133] updating tests

---
 test/acceptance/copy-endpoints.js | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 0e428a1d..a9acc8a7 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -6,11 +6,11 @@ const server = require('../../app/server')();
 const assert = require('../support/assert');
 
 describe('copy-endpoints', function() {
-    it('should works with copyfrom endpoint', function(done){
+    it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
                 sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
-                }),
+            }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
@@ -27,8 +27,7 @@ describe('copy-endpoints', function() {
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
                 sql: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
-                }),
-            
+            }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
@@ -47,8 +46,8 @@ describe('copy-endpoints', function() {
     it('should fail with copyfrom endpoint and without csv', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
-                }),
+                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+            }),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
         },{}, function(err, res) {
@@ -56,7 +55,7 @@ describe('copy-endpoints', function() {
             assert.deepEqual(
                 JSON.parse(res.body), 
                 {
-                    error:['The file is missing']
+                    error:['No rows copied']
                 }
             );
             done();
@@ -65,9 +64,7 @@ describe('copy-endpoints', function() {
 
     it('should fail with copyfrom endpoint and without sql', function(done){
         assert.response(server, {
-            url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
-                }),
+            url: "/api/v1/sql/copyfrom",
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),            
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
@@ -83,7 +80,7 @@ describe('copy-endpoints', function() {
         });
     });
 
-    it('should works with copyto endpoint', function(done){
+    it('should work with copyto endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyto?" + querystring.stringify({
                 sql: 'COPY copy_endpoints_test TO STDOUT',

From f749798ca523d0b963c9b94e1ce9d38bd2ce15d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 11 May 2018 10:38:33 +0200
Subject: [PATCH 054/133] validating inputs

---
 app/controllers/copy_controller.js | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index a11f53d3..5085e5a3 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -103,12 +103,19 @@ CopyController.prototype.responseCopyTo = function (req, res) {
 };
 
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
-    let sql = req.query.sql;
-        
-    // curl -vv -X POST --data-binary @test.csv "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom?api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961&sql=copy+foo+(i,t)+from+stdin+with+(format+csv,header+false)"
+    const { sql } = req.query;
 
-    // curl -vv -X POST --data-binary @test.csv "http://cdb.localhost.lan:8080/api/v2/sql/copyfrom?api_key=5b7056ebaa4bae42c0e99283785f0fd63e36e961&sql=copy+foo+(i,t)+from+stdin"
+    if (!sql) {	
+        return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
+    }
+
+    // Only accept SQL that starts with 'COPY'
+    if (!sql.toUpperCase().startsWith("COPY ")) {
+        return next(new Error("SQL must start with COPY"));
+    }
     
+    req.setEncoding('utf8');
+
     try {
         const start_time = Date.now();
 
@@ -132,7 +139,6 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
                 return next();
             });
 
-            // req is a readable stream (cool!)
             req.pipe(pgstream);
         });
 

From 791967877ccbbf5157aac1979b1d5d0c0dfbf393 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 11 May 2018 13:33:54 +0200
Subject: [PATCH 055/133] addining gzip support to copyfrom

---
 app/controllers/copy_controller.js      |  15 +++++++++------
 test/acceptance/copy-endpoints.js       |  21 +++++++++++++++++++++
 test/support/csv/copy_test_table.csv.gz | Bin 0 -> 96 bytes
 3 files changed, 30 insertions(+), 6 deletions(-)
 create mode 100644 test/support/csv/copy_test_table.csv.gz

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 5085e5a3..227cc02b 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -9,6 +9,7 @@ const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 
+const zlib = require('zlib');
 const PSQL = require('cartodb-psql');
 const copyTo = require('pg-copy-streams').to;
 const copyFrom = require('pg-copy-streams').from;
@@ -81,7 +82,7 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
             res.on('error', next);
             pgstream.on('error', next);
             pgstream.on('end', next);
-            
+
             pgstream.pipe(res);
         });
     } catch (err) {
@@ -92,7 +93,7 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
 
 CopyController.prototype.responseCopyTo = function (req, res) {
     let { filename } = req.query;
-    
+
     if (!filename) {
         filename = 'carto-sql-copyto.dmp';
     }
@@ -105,7 +106,7 @@ CopyController.prototype.responseCopyTo = function (req, res) {
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
     const { sql } = req.query;
 
-    if (!sql) {	
+    if (!sql) {
         return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
     }
 
@@ -113,8 +114,6 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
     if (!sql.toUpperCase().startsWith("COPY ")) {
         return next(new Error("SQL must start with COPY"));
     }
-    
-    req.setEncoding('utf8');
 
     try {
         const start_time = Date.now();
@@ -139,7 +138,11 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
                 return next();
             });
 
-            req.pipe(pgstream);
+            if (req.get('content-encoding') === 'gzip') {
+                req.pipe(zlib.createGunzip()).pipe(pgstream);
+            } else {
+                req.pipe(pgstream);
+            }
         });
 
     } catch (err) {
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index a9acc8a7..0efda5a2 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -98,4 +98,25 @@ describe('copy-endpoints', function() {
         });
     });
 
+    it('should work with copyfrom and gzip', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+                gzip: true
+            }),
+            data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv.gz'),
+            headers: {
+                host: 'vizzuality.cartodb.com', 
+                'content-encoding': 'gzip'
+            },
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            const response = JSON.parse(res.body);
+            assert.equal(!!response.time, true);
+            assert.strictEqual(response.total_rows, 6);
+            done();
+        });
+    });
+
 });
diff --git a/test/support/csv/copy_test_table.csv.gz b/test/support/csv/copy_test_table.csv.gz
new file mode 100644
index 0000000000000000000000000000000000000000..bac1a2d17a8c736b185da5e9f7fd274832ded14e
GIT binary patch
literal 96
zcmV-m0H6OKiwForLG@Yy17mM+d0%v8b97&HVPb4$E@N|c0Lx6#$xF;l<uWwX2}mr>
z;W9MR2}mtTE#fjX*6~d&DalAJ=Q1?W;W9MU@dBzV<}x(X@ygH0;{pH_5%|J60000`
CkSI3*

literal 0
HcmV?d00001


From d9707428609c111b22ccae8d47d723f4cf447418 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 11 May 2018 14:10:52 +0200
Subject: [PATCH 056/133] removing uneeded param

---
 test/acceptance/copy-endpoints.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 0efda5a2..db0a5c32 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -101,8 +101,7 @@ describe('copy-endpoints', function() {
     it('should work with copyfrom and gzip', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
-                gzip: true
+                sql: "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv.gz'),
             headers: {

From eee8bc267497460429ff180ce34d01f9b85fbcb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 11 May 2018 14:12:23 +0200
Subject: [PATCH 057/133] moving copyto headers to correct position

---
 app/controllers/copy_controller.js | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 227cc02b..6736ebd4 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -47,7 +47,6 @@ CopyController.prototype.route = function (app) {
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
             this.handleCopyTo.bind(this),
-            this.responseCopyTo.bind(this),
             errorMiddleware()
         ];
     };
@@ -57,7 +56,9 @@ CopyController.prototype.route = function (app) {
 };
 
 CopyController.prototype.handleCopyTo = function (req, res, next) {
-    const { sql } = req.query;
+    const { sql, filename = 'carto-sql-copyto.dmp' } = req.query;
+
+    console.log('request headers', req.headers)
 
     if (!sql) {
         throw new Error("Parameter 'sql' is missing");
@@ -83,6 +84,9 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
             pgstream.on('error', next);
             pgstream.on('end', next);
 
+            res.setHeader("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
+            res.setHeader("Content-Type", "application/octet-stream");
+
             pgstream.pipe(res);
         });
     } catch (err) {
@@ -91,18 +95,6 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
 
 };
 
-CopyController.prototype.responseCopyTo = function (req, res) {
-    let { filename } = req.query;
-
-    if (!filename) {
-        filename = 'carto-sql-copyto.dmp';
-    }
-
-    res.setHeader("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
-    res.setHeader("Content-Type", "application/octet-stream");
-    res.send();
-};
-
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
     const { sql } = req.query;
 

From f2acc8f6536a6c7fdfd62f968527a5309e18474d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 11 May 2018 14:16:26 +0200
Subject: [PATCH 058/133] forgotten log

---
 app/controllers/copy_controller.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 6736ebd4..96bb2708 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -58,8 +58,6 @@ CopyController.prototype.route = function (app) {
 CopyController.prototype.handleCopyTo = function (req, res, next) {
     const { sql, filename = 'carto-sql-copyto.dmp' } = req.query;
 
-    console.log('request headers', req.headers)
-
     if (!sql) {
         throw new Error("Parameter 'sql' is missing");
     }

From bc8a8673349a72d720763c6a2b7daf9bc25e443f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 11 May 2018 14:51:43 +0200
Subject: [PATCH 059/133] jshinty happy

---
 app/controllers/copy_controller.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 96bb2708..770a8ba5 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -56,7 +56,8 @@ CopyController.prototype.route = function (app) {
 };
 
 CopyController.prototype.handleCopyTo = function (req, res, next) {
-    const { sql, filename = 'carto-sql-copyto.dmp' } = req.query;
+    const { sql } = req.query;
+    const filename = req.query.filename || 'carto-sql-copyto.dmp';
 
     if (!sql) {
         throw new Error("Parameter 'sql' is missing");

From 9016647a6f66436b413d854d2a177f7454ca405f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 14 May 2018 18:31:44 +0200
Subject: [PATCH 060/133] reverting use of busboy instead of multer

---
 app/middlewares/body-parser.js | 27 ++---------------
 npm-shrinkwrap.json            | 54 ++++++++++++++++++++++++++++++++++
 package.json                   |  2 +-
 3 files changed, 58 insertions(+), 25 deletions(-)

diff --git a/app/middlewares/body-parser.js b/app/middlewares/body-parser.js
index dd704d0e..6c44e295 100644
--- a/app/middlewares/body-parser.js
+++ b/app/middlewares/body-parser.js
@@ -11,7 +11,7 @@
  */
 
 var qs = require('qs');
-const Busboy = require('busboy');
+var multer = require('multer');
 
 /**
  * Extract the mime type from the given request's
@@ -141,26 +141,5 @@ exports.parse['application/json'] = function(req, options, fn){
     });
 };
 
-
-exports.parse['multipart/form-data'] = function(req, options, fn){
-    if (req.method === 'POST') {
-        req.setEncoding('utf8');
-
-        const busboy = new Busboy({ 
-            headers: req.headers,
-            limits: {
-                files: 0
-            }
-        });
-
-        busboy.on('field', function(fieldname, val) {
-            req.body[fieldname] = val;
-        });
-
-        busboy.on('finish', function() {
-            fn();
-        });
-
-        req.pipe(busboy);
-    }
-};
+var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
+exports.parse['multipart/form-data'] = multipartMiddleware.none();
diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index 4386c33d..5e91e28a 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -17,6 +17,11 @@
       "from": "ansi-styles@>=2.2.1 <3.0.0",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz"
     },
+    "append-field": {
+      "version": "0.1.0",
+      "from": "append-field@>=0.1.0 <0.2.0",
+      "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz"
+    },
     "array-flatten": {
       "version": "1.1.1",
       "from": "array-flatten@1.1.1",
@@ -110,6 +115,11 @@
       "from": "brace-expansion@>=1.1.7 <2.0.0",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz"
     },
+    "buffer-from": {
+      "version": "1.0.0",
+      "from": "buffer-from@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.0.0.tgz"
+    },
     "buffer-writer": {
       "version": "1.0.1",
       "from": "buffer-writer@1.0.1",
@@ -185,6 +195,33 @@
       "from": "concat-map@0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz"
     },
+    "concat-stream": {
+      "version": "1.6.2",
+      "from": "concat-stream@>=1.5.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+      "dependencies": {
+        "isarray": {
+          "version": "1.0.0",
+          "from": "isarray@>=1.0.0 <1.1.0",
+          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
+        },
+        "process-nextick-args": {
+          "version": "2.0.0",
+          "from": "process-nextick-args@>=2.0.0 <2.1.0",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz"
+        },
+        "readable-stream": {
+          "version": "2.3.6",
+          "from": "readable-stream@>=2.2.2 <3.0.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz"
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "from": "string_decoder@>=1.1.1 <1.2.0",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz"
+        }
+      }
+    },
     "content-disposition": {
       "version": "0.5.1",
       "from": "content-disposition@0.5.1",
@@ -645,6 +682,18 @@
       "from": "ms@0.7.1",
       "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
     },
+    "multer": {
+      "version": "1.2.1",
+      "from": "multer@>=1.2.0 <1.3.0",
+      "resolved": "https://registry.npmjs.org/multer/-/multer-1.2.1.tgz",
+      "dependencies": {
+        "object-assign": {
+          "version": "3.0.0",
+          "from": "object-assign@>=3.0.0 <4.0.0",
+          "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-3.0.0.tgz"
+        }
+      }
+    },
     "mv": {
       "version": "2.1.1",
       "from": "mv@>=2.0.0 <3.0.0",
@@ -1093,6 +1142,11 @@
       "from": "type-is@>=1.6.6 <1.7.0",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz"
     },
+    "typedarray": {
+      "version": "0.0.6",
+      "from": "typedarray@>=0.0.6 <0.0.7",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz"
+    },
     "underscore": {
       "version": "1.6.0",
       "from": "underscore@>=1.6.0 <1.7.0",
diff --git a/package.json b/package.json
index 90559328..f6947749 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,6 @@
     "basic-auth": "^2.0.0",
     "bintrees": "1.0.1",
     "bunyan": "1.8.1",
-    "busboy": "^0.2.14",
     "cartodb-psql": "0.10.1",
     "cartodb-query-tables": "0.2.0",
     "cartodb-redis": "1.0.0",
@@ -28,6 +27,7 @@
     "express": "~4.13.3",
     "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "~2.5.0",
+    "multer": "~1.2.0",	
     "node-statsd": "~0.0.7",
     "node-uuid": "^1.4.7",
     "oauth-client": "0.3.0",

From 15fd3bdfee0a2fd3a9268be08551fbebb6fb7b78 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andyesch@protonmail.com>
Date: Mon, 14 May 2018 12:38:00 -0400
Subject: [PATCH 061/133] updates python snippets to be python 3 compatible

---
 doc/copy_queries.md | 85 +++++++++++++++++++++++----------------------
 1 file changed, 44 insertions(+), 41 deletions(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 035b763b..8a64a936 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -22,7 +22,7 @@ If the `COPY` command, the supplied file, and the target table do not all match,
 * `sql` provided either in the request URL parameters or in a multipart form variable.
 * `file` provided as a multipart form variable; this is the actual file content, not a filename.
 
-Composing a multipart form data upload is moderately complicated, so almost all developers will use a tool or scripting language to upload data to CARTO via "copy from". 
+Composing a multipart form data upload is moderately complicated, so almost all developers will use a tool or scripting language to upload data to CARTO via "copy from".
 
 ### Example
 
@@ -34,7 +34,7 @@ For a table to be readable by CARTO, it must have a minimum of three columns wit
 
 Creating a new CARTO table with all the right triggers and columns can be tricky, so here is an example:
 
-    -- create the table using the *required* columns and a 
+    -- create the table using the *required* columns and a
     -- couple more
     CREATE TABLE upload_example (
         the_geom geometry,
@@ -45,7 +45,7 @@ Creating a new CARTO table with all the right triggers and columns can be tricky
     -- adds the 'cartodb_id' and 'the_geom_webmercator'
     -- adds the required triggers and indexes
     SELECT CDB_CartodbfyTable('upload_example');
-    
+
 Now you are ready to upload your file. Suppose you have a CSV file like this:
 
     the_geom,name,age
@@ -55,7 +55,7 @@ Now you are ready to upload your file. Suppose you have a CSV file like this:
 
 The `COPY` command to upload this file needs to specify the file format (CSV), the fact that there is a header line before the actual data begins, and to enumerate the columns that are in the file so they can be matched to the table columns.
 
-    COPY upload_example (the_geom, name, age) 
+    COPY upload_example (the_geom, name, age)
     FROM STDIN WITH (FORMAT csv, HEADER true)
 
 The `FROM STDIN` option tells the database that the input will come from a data stream, and the SQL API will read our uploaded file and use it to feed the stream.
@@ -86,24 +86,26 @@ To upload a larger file, using compression for a faster transfer, first compress
 
 ### Python Example
 
-The [Requests](http://docs.python-requests.org/en/master/user/quickstart/) library for HTTP makes doing a file upload relatively terse. 
+The [Requests](http://docs.python-requests.org/en/master/user/quickstart/) library for HTTP makes doing a file upload relatively terse.
 
-    import requests
+```python
+import requests
 
-    api_key = {api_key}
-    username = {api_key}
-    upload_file = 'upload_example.csv'
-    sql = "COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)"
+api_key = {api_key}
+username = {api_key}
+upload_file = 'upload_example.csv'
+sql = "COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)"
 
-    url = "http://%s.carto.com/api/v2/sql/copyfrom" % username    
-    with open(upload_file, 'rb') as f:
-        r = requests.post(url, params={'api_key':api_key, 'sql':sql}, data=f, stream=True)
+url = "http://%s.carto.com/api/v2/sql/copyfrom" % username
+with open(upload_file, 'rb') as f:
+    r = requests.post(url, params={'api_key': api_key, 'sql': sql}, data=f, stream=True)
 
-        if r.status_code != 200:
-            print r.text
-        else:
-            status = r.json()
-            print "Success: %s rows imported" % status['total_rows']
+    if r.status_code != 200:
+        print(r.text)
+    else:
+        status = r.json()
+        print("Success: %s rows imported" % status['total_rows'])
+```
 
 A slightly more sophisticated script could read the headers from the CSV and compose the `COPY` command on the fly.
 
@@ -116,7 +118,7 @@ A successful upload will return with status code 200, and a small JSON with info
 A failed upload will return with status code 400 and a larger JSON with the PostgreSQL error string, and a stack trace from the SQL API.
 
     {"error":["Unexpected field"]}
-    
+
 ## Copy To
 
 "Copy to" copies data "to" your desired output file, "from" CARTO.
@@ -145,7 +147,7 @@ The SQL to start a "copy to" can specify
 For our example, we'll read back just the three columns we originally loaded:
 
     COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)
-    
+
 The SQL needs to be URL-encoded before being embedded in the CURL command, so the final result looks like this:
 
     curl \
@@ -157,28 +159,29 @@ The SQL needs to be URL-encoded before being embedded in the CURL command, so th
 
 The Python to "copy to" is very simple, because the HTTP call is a simple get. The only complexity in this example is at the end, where the result is streamed back block-by-block, to avoid pulling the entire download into memory before writing to file.
 
-    import requests
-    import re
+```python
+import requests
+import re
 
-    api_key = {api_key}
-    username = {api_key}
-    download_file = 'upload_example_dl.csv'
-    sql = "COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)"
+api_key = {api_key}
+username = {api_key}
+download_file = 'upload_example_dl.csv'
+sql = "COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)"
 
-    # request the download, specifying desired file name
-    url = "http://%s.carto.com/api/v2/sql/copyto" % username    
-    r = requests.get(url, params={'api_key':api_key, 'sql':sql, 'filename':download_file}, stream=True)
-    r.raise_for_status()
+# request the download, specifying desired file name
+url = "http://%s.carto.com/api/v2/sql/copyto" % username
+r = requests.get(url, params={'api_key': api_key, 'sql': sql, 'filename': download_file}, stream=True)
+r.raise_for_status()
 
-    # read save file name from response headers
-    d = r.headers['content-disposition']
-    savefilename = re.findall("filename=(.+)", d)
-
-    if len(savefilename) > 0:
-        with open(savefilename[0], 'wb') as handle:
-            for block in r.iter_content(1024):
-                handle.write(block)        
-        print "Downloaded to: %s" % savefilename
-    else:
-        print "Error: could not find read file name from headers"
+# read save file name from response headers
+d = r.headers['content-disposition']
+savefilename = re.findall("filename=(.+)", d)
 
+if len(savefilename) > 0:
+    with open(savefilename[0], 'wb') as handle:
+        for block in r.iter_content(1024):
+            handle.write(block)
+    print("Downloaded to: %s" % savefilename)
+else:
+    print("Error: could not find read file name from headers")
+```

From 82c9fec18a5d908464f99f1ebc4d7060186b5789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 18 May 2018 17:31:29 +0200
Subject: [PATCH 062/133] copy from metrics

---
 app/controllers/copy_controller.js | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 770a8ba5..cd35a721 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -106,6 +106,8 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
         return next(new Error("SQL must start with COPY"));
     }
 
+    res.locals.copyFromSize = 0;
+
     try {
         const start_time = Date.now();
 
@@ -130,9 +132,14 @@ CopyController.prototype.handleCopyFrom = function (req, res, next) {
             });
 
             if (req.get('content-encoding') === 'gzip') {
-                req.pipe(zlib.createGunzip()).pipe(pgstream);
+                req
+                    .pipe(zlib.createGunzip())
+                    .on('data', data => res.locals.copyFromSize += data.length)
+                    .pipe(pgstream);
             } else {
-                req.pipe(pgstream);
+                req
+                    .on('data', data => res.locals.copyFromSize += data.length)
+                    .pipe(pgstream);
             }
         });
 
@@ -147,6 +154,18 @@ CopyController.prototype.responseCopyFrom = function (req, res, next) {
         return next(new Error("No rows copied"));
     }
 
+    if (req.profiler) {
+        const copyFromMetrics = {
+            time: res.body.time, //seconds
+            size: res.locals.copyFromSize, //bytes
+            total_rows: res.body.total_rows, 
+            gzip: req.get('content-encoding') === 'gzip'
+        };
+
+        req.profiler.add({ copyFrom: copyFromMetrics });
+        res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());+
+    }
+
     res.send(res.body);
 };
 

From 816779fefde953834448b75b0ad0572185641911 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 18 May 2018 17:32:59 +0200
Subject: [PATCH 063/133] erratum

---
 app/controllers/copy_controller.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index cd35a721..57615dbd 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -156,14 +156,14 @@ CopyController.prototype.responseCopyFrom = function (req, res, next) {
 
     if (req.profiler) {
         const copyFromMetrics = {
-            time: res.body.time, //seconds
             size: res.locals.copyFromSize, //bytes
+            time: res.body.time, //seconds
             total_rows: res.body.total_rows, 
             gzip: req.get('content-encoding') === 'gzip'
         };
 
         req.profiler.add({ copyFrom: copyFromMetrics });
-        res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());+
+        res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
     }
 
     res.send(res.body);

From 64fc0c32e3f31f26a6edc6cf6066f714e56043d5 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Mon, 21 May 2018 07:47:16 -0700
Subject: [PATCH 064/133] Change from 'sql' as query parameter to 'q', aping
 existing sql api

---
 app/controllers/copy_controller.js |  8 ++++----
 doc/copy_queries.md                | 30 +++++++++++++++---------------
 test/acceptance/copy-endpoints.js  | 14 +++++++-------
 3 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 770a8ba5..f1dd893d 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -56,11 +56,11 @@ CopyController.prototype.route = function (app) {
 };
 
 CopyController.prototype.handleCopyTo = function (req, res, next) {
-    const { sql } = req.query;
+    const { q: sql } = req.query;
     const filename = req.query.filename || 'carto-sql-copyto.dmp';
 
     if (!sql) {
-        throw new Error("Parameter 'sql' is missing");
+        throw new Error("Parameter 'q' is missing");
     }
 
     // Only accept SQL that starts with 'COPY'
@@ -95,10 +95,10 @@ CopyController.prototype.handleCopyTo = function (req, res, next) {
 };
 
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
-    const { sql } = req.query;
+    const { q: sql } = req.query;
 
     if (!sql) {
-        return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
+        return next(new Error("Parameter 'q' is missing, must be in URL or first field in POST"));
     }
 
     // Only accept SQL that starts with 'COPY'
diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 035b763b..8e737bea 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -16,13 +16,13 @@ The PostgreSQL `COPY` command is extremely fast, but requires very precise input
 
 If the `COPY` command, the supplied file, and the target table do not all match, the upload will fail.
 
-"Copy from" copies data "from" your file, "to" CARTO. "Copy from" uses [multipart/form-data](https://stackoverflow.com/questions/8659808/how-does-http-file-upload-work) to stream an upload file to the server. This avoids limitations around file size and any need for temporary storage: the data travels from your file straight into the database.
+"Copy from" copies data "from" your file, "to" CARTO. "Copy from" uses chunked encoding (`Transfer-Encoding: chunked`) to stream an upload file to the server. This avoids limitations around file size and any need for temporary storage: the data travels from your file straight into the database.
 
 * `api_key` provided in the request URL parameters.
-* `sql` provided either in the request URL parameters or in a multipart form variable.
-* `file` provided as a multipart form variable; this is the actual file content, not a filename.
+* `q` the copy SQL provided either in the request URL parameters.
+* the actual copy file content, as the body of the POST.
 
-Composing a multipart form data upload is moderately complicated, so almost all developers will use a tool or scripting language to upload data to CARTO via "copy from". 
+Composing a chunked POST is moderately complicated, so most developers will use a tool or scripting language to upload data to CARTO via "copy from". 
 
 ### Example
 
@@ -60,19 +60,19 @@ The `COPY` command to upload this file needs to specify the file format (CSV), t
 
 The `FROM STDIN` option tells the database that the input will come from a data stream, and the SQL API will read our uploaded file and use it to feed the stream.
 
-To actually run upload, you will need a tool or script that can generate a `multipart/form-data` POST, so here are a few examples in different languages.
+To actually run upload, you will need a tool or script that can generate a chunked POST, so here are a few examples in different languages.
 
 ### CURL Example
 
-The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from the command-line, and supports multi-part file upload, so it can feed the `copyfrom` end point.
+The [curl](https://curl.haxx.se/) utility makes it easy to run web requests from the command-line, and supports chunked POST upload, so it can feed the `copyfrom` end point.
 
 Assuming that you have already created the table, and that the CSV file is named "upload_example.csv":
 
     curl -X POST \
-        --data-binary @upload_example.csv \
         -H "Transfer-Encoding: chunked" \
         -H "Content-Type: application/octet-stream" \
-        "http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
+        --data-binary @upload_example.csv \
+        "http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&q=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
 
 To upload a larger file, using compression for a faster transfer, first compress the file, and then upload it with the content encoding set:
 
@@ -81,7 +81,7 @@ To upload a larger file, using compression for a faster transfer, first compress
         -H "Transfer-Encoding: chunked" \
         -H "Content-Type: application/octet-stream" \
         --data-binary @upload_example.csv.gz \
-        "http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&sql=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
+        "http://{username}.carto.com/api/v2/sql/copyfrom?api_key={api_key}&q=COPY+upload_example+(the_geom,+name,+age)+FROM+STDIN+WITH+(FORMAT+csv,+HEADER+true)"
 
 
 ### Python Example
@@ -93,11 +93,11 @@ The [Requests](http://docs.python-requests.org/en/master/user/quickstart/) libra
     api_key = {api_key}
     username = {api_key}
     upload_file = 'upload_example.csv'
-    sql = "COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)"
+    q = "COPY upload_example (the_geom, name, age) FROM STDIN WITH (FORMAT csv, HEADER true)"
 
     url = "http://%s.carto.com/api/v2/sql/copyfrom" % username    
     with open(upload_file, 'rb') as f:
-        r = requests.post(url, params={'api_key':api_key, 'sql':sql}, data=f, stream=True)
+        r = requests.post(url, params={'api_key':api_key, 'q':q}, data=f, stream=True)
 
         if r.status_code != 200:
             print r.text
@@ -129,7 +129,7 @@ Using the `copyto` end point to extract data bypasses the usual JSON formatting
 
 "Copy to" is a simple HTTP GET end point, so any tool or language can be easily used to download data, supplying the following parameters in the URL:
 
-* `sql`, the "COPY" command to extract the data.
+* `q`, the "COPY" command to extract the data.
 * `filename`, the filename to put in the "Content-disposition" HTTP header. Useful for tools that automatically save the download to a file name.
 * `api_key`, your API key for reading non-public tables.
 
@@ -151,7 +151,7 @@ The SQL needs to be URL-encoded before being embedded in the CURL command, so th
     curl \
         --output upload_example_dl.csv \
 		--compressed \
-        "http://{username}.carto.com/api/v2/sql/copyto?sql=COPY+upload_example+(the_geom,name,age)+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
+        "http://{username}.carto.com/api/v2/sql/copyto?q=COPY+upload_example+(the_geom,name,age)+TO+stdout+WITH(FORMAT+csv,HEADER+true)&api_key={api_key}"
 
 ### Python Example
 
@@ -163,11 +163,11 @@ The Python to "copy to" is very simple, because the HTTP call is a simple get. T
     api_key = {api_key}
     username = {api_key}
     download_file = 'upload_example_dl.csv'
-    sql = "COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)"
+    q = "COPY upload_example (the_geom, name, age) TO stdout WITH (FORMAT csv, HEADER true)"
 
     # request the download, specifying desired file name
     url = "http://%s.carto.com/api/v2/sql/copyto" % username    
-    r = requests.get(url, params={'api_key':api_key, 'sql':sql, 'filename':download_file}, stream=True)
+    r = requests.get(url, params={'api_key':api_key, 'q':q, 'filename':download_file}, stream=True)
     r.raise_for_status()
 
     # read save file name from response headers
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index db0a5c32..48c8edb6 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -9,7 +9,7 @@ describe('copy-endpoints', function() {
     it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
@@ -26,7 +26,7 @@ describe('copy-endpoints', function() {
     it('should fail with copyfrom endpoint and unexisting table', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
@@ -46,7 +46,7 @@ describe('copy-endpoints', function() {
     it('should fail with copyfrom endpoint and without csv', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
@@ -62,7 +62,7 @@ describe('copy-endpoints', function() {
         });
     });
 
-    it('should fail with copyfrom endpoint and without sql', function(done){
+    it('should fail with copyfrom endpoint and without q', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom",
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),            
@@ -73,7 +73,7 @@ describe('copy-endpoints', function() {
             assert.deepEqual(
                 JSON.parse(res.body), 
                 {
-                    error:["Parameter 'sql' is missing, must be in URL or first field in POST"]
+                    error:["Parameter 'q' is missing, must be in URL or first field in POST"]
                 }
             );
             done();
@@ -83,7 +83,7 @@ describe('copy-endpoints', function() {
     it('should work with copyto endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyto?" + querystring.stringify({
-                sql: 'COPY copy_endpoints_test TO STDOUT',
+                q: 'COPY copy_endpoints_test TO STDOUT',
                 filename: '/tmp/output.dmp'
             }),
             headers: {host: 'vizzuality.cartodb.com'},
@@ -101,7 +101,7 @@ describe('copy-endpoints', function() {
     it('should work with copyfrom and gzip', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv.gz'),
             headers: {

From 81be15fbc3c46e173ee62fab674ec89b9e17e971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 21 May 2018 19:13:44 +0200
Subject: [PATCH 065/133] adding format to copy metrics

---
 app/controllers/copy_controller.js |  2 ++
 app/utils/query_info.js            | 21 +++++++++++++
 test/acceptance/copy-endpoints.js  | 22 ++++++++++++++
 test/unit/query_info.test.js       | 49 ++++++++++++++++++++++++++++++
 4 files changed, 94 insertions(+)
 create mode 100644 app/utils/query_info.js
 create mode 100644 test/unit/query_info.test.js

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 57615dbd..ac1fdafd 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -8,6 +8,7 @@ const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
 const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
+const { getFormatFromCopyQuery } = require('../utils/query_info');
 
 const zlib = require('zlib');
 const PSQL = require('cartodb-psql');
@@ -157,6 +158,7 @@ CopyController.prototype.responseCopyFrom = function (req, res, next) {
     if (req.profiler) {
         const copyFromMetrics = {
             size: res.locals.copyFromSize, //bytes
+            format: getFormatFromCopyQuery(req.query.sql),
             time: res.body.time, //seconds
             total_rows: res.body.total_rows, 
             gzip: req.get('content-encoding') === 'gzip'
diff --git a/app/utils/query_info.js b/app/utils/query_info.js
new file mode 100644
index 00000000..dd39e5f8
--- /dev/null
+++ b/app/utils/query_info.js
@@ -0,0 +1,21 @@
+const COPY_FORMATS = ['TEXT', 'CSV', 'BINARY'];
+
+module.exports = {
+    getFormatFromCopyQuery(copyQuery) {
+        let format = 'TEXT'; // Postgres default format
+
+        copyQuery = copyQuery.toUpperCase();
+
+        if (!copyQuery.toUpperCase().startsWith("COPY ")) {
+            return false;
+        }
+       
+        const regex = /(\bFORMAT\s+)(\w+)/;
+        const result = regex.exec(copyQuery);
+        if (result && result.length >= 3 && COPY_FORMATS.includes(result[2])) {
+            format = result[2];
+        }
+
+        return format;
+    }
+};
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index db0a5c32..e5ad87e7 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -19,6 +19,17 @@ describe('copy-endpoints', function() {
             const response = JSON.parse(res.body);
             assert.equal(!!response.time, true);
             assert.strictEqual(response.total_rows, 6);
+
+            assert.ok(res.headers['x-sqlapi-profiler']);
+            const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
+            assert.ok(headers.copyFrom);
+            const metrics = headers.copyFrom;
+            assert.equal(metrics.size, 57);
+            assert.equal(metrics.format, 'CSV');
+            assert.equal(metrics.time, response.time);
+            assert.equal(metrics.total_rows, response.total_rows);
+            assert.equal(metrics.gzip, false);
+
             done();
         });
     });
@@ -114,6 +125,17 @@ describe('copy-endpoints', function() {
             const response = JSON.parse(res.body);
             assert.equal(!!response.time, true);
             assert.strictEqual(response.total_rows, 6);
+
+            assert.ok(res.headers['x-sqlapi-profiler']);
+            const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
+            assert.ok(headers.copyFrom);
+            const metrics = headers.copyFrom;
+            assert.equal(metrics.size, 57);
+            assert.equal(metrics.format, 'CSV');
+            assert.equal(metrics.time, response.time);
+            assert.equal(metrics.total_rows, response.total_rows);
+            assert.equal(metrics.gzip, true);
+
             done();
         });
     });
diff --git a/test/unit/query_info.test.js b/test/unit/query_info.test.js
new file mode 100644
index 00000000..96649ee4
--- /dev/null
+++ b/test/unit/query_info.test.js
@@ -0,0 +1,49 @@
+const assert = require('assert');
+const queryInfo = require('../../app/utils/query_info');
+
+describe('query info', function () {
+    describe('copy format', function() {
+        describe('csv', function() {
+            const csvValidQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT  CSV, DELIMITER ',', HEADER true)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV , DELIMITER ',', HEADER true)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV)",
+            ];
+
+            csvValidQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.equal(result, 'CSV');
+                });
+            });
+        });
+
+        describe('text', function() {
+            const csvValidQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT TEXT)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN",
+            ];
+
+            csvValidQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.equal(result, 'TEXT');
+                });
+            });
+        });
+
+        describe('text', function() {
+            const csvValidQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT BINARY)",
+            ];
+
+            csvValidQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.equal(result, 'BINARY');
+                });
+            });
+        });
+    });
+});

From 433bd01c27a91dcfc38ee8539d4c0590818a347e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 11:54:10 +0200
Subject: [PATCH 066/133] copyto metrics

---
 app/controllers/copy_controller.js | 77 +++++++++++++++++-------------
 app/server.js                      |  3 +-
 test/acceptance/copy-endpoints.js  | 20 +++++++-
 3 files changed, 64 insertions(+), 36 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index ac1fdafd..1d18cb17 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -16,10 +16,11 @@ const copyTo = require('pg-copy-streams').to;
 const copyFrom = require('pg-copy-streams').from;
 
 
-function CopyController(metadataBackend, userDatabaseService, userLimitsService) {
+function CopyController(metadataBackend, userDatabaseService, userLimitsService, statsClient) {
     this.metadataBackend = metadataBackend;
     this.userDatabaseService = userDatabaseService;
     this.userLimitsService = userLimitsService;
+    this.statsClient = statsClient;
 }
 
 CopyController.prototype.route = function (app) {
@@ -47,7 +48,7 @@ CopyController.prototype.route = function (app) {
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
-            this.handleCopyTo.bind(this),
+            handleCopyTo(this.statsClient),
             errorMiddleware()
         ];
     };
@@ -56,44 +57,54 @@ CopyController.prototype.route = function (app) {
     app.get(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO));
 };
 
-CopyController.prototype.handleCopyTo = function (req, res, next) {
-    const { sql } = req.query;
-    const filename = req.query.filename || 'carto-sql-copyto.dmp';
+function handleCopyTo (statsClient) {
+    return function handleCopyToMiddleware (req, res, next) {
+        const { sql } = req.query;
+        const filename = req.query.filename || 'carto-sql-copyto.dmp';
 
-    if (!sql) {
-        throw new Error("Parameter 'sql' is missing");
-    }
+        if (!sql) {
+            throw new Error("Parameter 'sql' is missing");
+        }
 
-    // Only accept SQL that starts with 'COPY'
-    if (!sql.toUpperCase().startsWith("COPY ")) {
-        throw new Error("SQL must start with COPY");
-    }
+        // Only accept SQL that starts with 'COPY'
+        if (!sql.toUpperCase().startsWith("COPY ")) {
+            throw new Error("SQL must start with COPY");
+        }
 
-    try {
-        // Open pgsql COPY pipe and stream out to HTTP response
-        const pg = new PSQL(res.locals.userDbParams);
-        pg.connect(function (err, client) {
-            if (err) {
-                return next(err);
-            }
+        let metrics = {
+            size: 0,
+            time: null,
+            format: getFormatFromCopyQuery(req.query.sql)
+        };
 
-            let copyToStream = copyTo(sql);
-            const pgstream = client.query(copyToStream);
+        res.header("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
+        res.header("Content-Type", "application/octet-stream");
 
-            res.on('error', next);
-            pgstream.on('error', next);
-            pgstream.on('end', next);
+        try {
+            const start_time = Date.now();
 
-            res.setHeader("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
-            res.setHeader("Content-Type", "application/octet-stream");
+            // Open pgsql COPY pipe and stream out to HTTP response
+            const pg = new PSQL(res.locals.userDbParams);
+            pg.connect(function (err, client) {
+                if (err) {
+                    return next(err);
+                }
 
-            pgstream.pipe(res);
-        });
-    } catch (err) {
-        next(err);
-    }
-
-};
+                const pgstream = client.query(copyTo(sql));
+                pgstream
+                    .on('error', next)
+                    .on('data', data => metrics.size += data.length)
+                    .on('end', () => {
+                        metrics.time = (Date.now() - start_time) / 1000;
+                        statsClient.set('copyTo', JSON.stringify(metrics));
+                    })
+                    .pipe(res);
+            });
+        } catch (err) {
+            next(err);
+        }
+    };
+}
 
 CopyController.prototype.handleCopyFrom = function (req, res, next) {
     const { sql } = req.query;
diff --git a/app/server.js b/app/server.js
index bbd079f6..c005346c 100644
--- a/app/server.js
+++ b/app/server.js
@@ -178,7 +178,8 @@ function App(statsClient) {
     var copyController = new CopyController(
         metadataBackend, 
         userDatabaseService, 
-        userLimitsService
+        userLimitsService,
+        statsClient
     );
     copyController.route(app);
     
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index e5ad87e7..d72d2817 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -2,10 +2,22 @@ require('../helper');
 
 const fs = require('fs');
 const querystring = require('querystring');
-const server = require('../../app/server')();
 const assert = require('../support/assert');
+const os = require('os');
 
-describe('copy-endpoints', function() {
+const StatsClient = require('../../app/stats/client');
+if (global.settings.statsd) {
+    // Perform keyword substitution in statsd
+    if (global.settings.statsd.prefix) {
+        const hostToken = os.hostname().split('.').reverse().join('.');
+        global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, hostToken);
+    }
+}
+const statsClient = StatsClient.getInstance(global.settings.statsd);
+const server = require('../../app/server')(statsClient);
+
+
+describe.only('copy-endpoints', function() {
     it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
@@ -105,6 +117,10 @@ describe('copy-endpoints', function() {
                 res.body, 
                 '11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\tJohn\t10\n'
             );
+
+            assert.equal(res.headers['content-disposition'], 'attachment; filename=%2Ftmp%2Foutput.dmp');
+            assert.equal(res.headers['content-type'], 'application/octet-stream');
+
             done();
         });
     });

From f01191472bdc07a5f8a7a2ca858c4655fb4a364f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 11:56:50 +0200
Subject: [PATCH 067/133] refactoring copy controller middlewares

---
 app/controllers/copy_controller.js | 131 +++++++++++++++--------------
 1 file changed, 67 insertions(+), 64 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 1d18cb17..dd927194 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -34,8 +34,8 @@ CopyController.prototype.route = function (app) {
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
-            this.handleCopyFrom.bind(this),
-            this.responseCopyFrom.bind(this),
+            handleCopyFrom(),
+            responseCopyFrom(),
             errorMiddleware()
         ];
     };
@@ -106,80 +106,83 @@ function handleCopyTo (statsClient) {
     };
 }
 
-CopyController.prototype.handleCopyFrom = function (req, res, next) {
-    const { sql } = req.query;
+function handleCopyFrom () {
+    return function handleCopyFromMiddleware (req, res, next) {
+        const { sql } = req.query;
 
-    if (!sql) {
-        return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
-    }
+        if (!sql) {
+            return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
+        }
 
-    // Only accept SQL that starts with 'COPY'
-    if (!sql.toUpperCase().startsWith("COPY ")) {
-        return next(new Error("SQL must start with COPY"));
-    }
+        // Only accept SQL that starts with 'COPY'
+        if (!sql.toUpperCase().startsWith("COPY ")) {
+            return next(new Error("SQL must start with COPY"));
+        }
 
-    res.locals.copyFromSize = 0;
+        res.locals.copyFromSize = 0;
 
-    try {
-        const start_time = Date.now();
+        try {
+            const start_time = Date.now();
 
-        // Connect and run the COPY
-        const pg = new PSQL(res.locals.userDbParams);
-        pg.connect(function (err, client) {
-            if (err) {
-                return next(err);
-            }
+            // Connect and run the COPY
+            const pg = new PSQL(res.locals.userDbParams);
+            pg.connect(function (err, client) {
+                if (err) {
+                    return next(err);
+                }
 
-            let copyFromStream = copyFrom(sql);
-            const pgstream = client.query(copyFromStream);
-            pgstream.on('error', next);
-            pgstream.on('end', function () {
-                const end_time = Date.now();
-                res.body = {
-                    time: (end_time - start_time) / 1000,
-                    total_rows: copyFromStream.rowCount
-                };
+                let copyFromStream = copyFrom(sql);
+                const pgstream = client.query(copyFromStream);
+                pgstream.on('error', next);
+                pgstream.on('end', function () {
+                    const end_time = Date.now();
+                    res.body = {
+                        time: (end_time - start_time) / 1000,
+                        total_rows: copyFromStream.rowCount
+                    };
 
-                return next();
+                    return next();
+                });
+
+                if (req.get('content-encoding') === 'gzip') {
+                    req
+                        .pipe(zlib.createGunzip())
+                        .on('data', data => res.locals.copyFromSize += data.length)
+                        .pipe(pgstream);
+                } else {
+                    req
+                        .on('data', data => res.locals.copyFromSize += data.length)
+                        .pipe(pgstream);
+                }
             });
 
-            if (req.get('content-encoding') === 'gzip') {
-                req
-                    .pipe(zlib.createGunzip())
-                    .on('data', data => res.locals.copyFromSize += data.length)
-                    .pipe(pgstream);
-            } else {
-                req
-                    .on('data', data => res.locals.copyFromSize += data.length)
-                    .pipe(pgstream);
-            }
-        });
+        } catch (err) {
+            next(err);
+        }
+    };
+}
 
-    } catch (err) {
-        next(err);
-    }
+function responseCopyFrom () {
+    return function responseCopyFromMiddleware (req, res, next) {
+        if (!res.body || !res.body.total_rows) {
+            return next(new Error("No rows copied"));
+        }
 
-};
+        if (req.profiler) {
+            const copyFromMetrics = {
+                size: res.locals.copyFromSize, //bytes
+                format: getFormatFromCopyQuery(req.query.sql),
+                time: res.body.time, //seconds
+                total_rows: res.body.total_rows, 
+                gzip: req.get('content-encoding') === 'gzip'
+            };
 
-CopyController.prototype.responseCopyFrom = function (req, res, next) {
-    if (!res.body || !res.body.total_rows) {
-        return next(new Error("No rows copied"));
-    }
+            req.profiler.add({ copyFrom: copyFromMetrics });
+            res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
+        }
 
-    if (req.profiler) {
-        const copyFromMetrics = {
-            size: res.locals.copyFromSize, //bytes
-            format: getFormatFromCopyQuery(req.query.sql),
-            time: res.body.time, //seconds
-            total_rows: res.body.total_rows, 
-            gzip: req.get('content-encoding') === 'gzip'
-        };
-
-        req.profiler.add({ copyFrom: copyFromMetrics });
-        res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
-    }
-
-    res.send(res.body);
-};
+        res.send(res.body);
+    };
+}
 
 module.exports = CopyController;

From 79383bb119f03897d43a4d5657b38f128cfb7707 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 11:57:10 +0200
Subject: [PATCH 068/133] removing only

---
 test/acceptance/copy-endpoints.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index d72d2817..7f58d790 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -17,7 +17,7 @@ const statsClient = StatsClient.getInstance(global.settings.statsd);
 const server = require('../../app/server')(statsClient);
 
 
-describe.only('copy-endpoints', function() {
+describe('copy-endpoints', function() {
     it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({

From 86e0ebc14878537ae5cd68dedb8f97436dafc5e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 12:02:41 +0200
Subject: [PATCH 069/133] small refactor in copy format regex

---
 app/utils/query_info.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/utils/query_info.js b/app/utils/query_info.js
index dd39e5f8..b5815063 100644
--- a/app/utils/query_info.js
+++ b/app/utils/query_info.js
@@ -10,10 +10,10 @@ module.exports = {
             return false;
         }
        
-        const regex = /(\bFORMAT\s+)(\w+)/;
+        const regex = /\bFORMAT\s+(\w+)/;
         const result = regex.exec(copyQuery);
-        if (result && result.length >= 3 && COPY_FORMATS.includes(result[2])) {
-            format = result[2];
+        if (result && result.length === 2 && COPY_FORMATS.includes(result[1])) {
+            format = result[1];
         }
 
         return format;

From f31f00dbbc8e1c6b0742197e639e96c972eaa75e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 12:05:16 +0200
Subject: [PATCH 070/133] fix test vars naming

---
 test/unit/query_info.test.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/unit/query_info.test.js b/test/unit/query_info.test.js
index 96649ee4..97202c04 100644
--- a/test/unit/query_info.test.js
+++ b/test/unit/query_info.test.js
@@ -4,14 +4,14 @@ const queryInfo = require('../../app/utils/query_info');
 describe('query info', function () {
     describe('copy format', function() {
         describe('csv', function() {
-            const csvValidQueries = [
+            const validQueries = [
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT  CSV, DELIMITER ',', HEADER true)",
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV , DELIMITER ',', HEADER true)",
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV)",
             ];
 
-            csvValidQueries.forEach(query => {
+            validQueries.forEach(query => {
                 it(query, function() {
                     const result = queryInfo.getFormatFromCopyQuery(query);
                     assert.equal(result, 'CSV');
@@ -20,12 +20,12 @@ describe('query info', function () {
         });
 
         describe('text', function() {
-            const csvValidQueries = [
+            const validQueries = [
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT TEXT)",
                 "COPY copy_endpoints_test (id, name) FROM STDIN",
             ];
 
-            csvValidQueries.forEach(query => {
+            validQueries.forEach(query => {
                 it(query, function() {
                     const result = queryInfo.getFormatFromCopyQuery(query);
                     assert.equal(result, 'TEXT');
@@ -33,12 +33,12 @@ describe('query info', function () {
             });
         });
 
-        describe('text', function() {
-            const csvValidQueries = [
+        describe('binary', function() {
+            const validQueries = [
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT BINARY)",
             ];
 
-            csvValidQueries.forEach(query => {
+            validQueries.forEach(query => {
                 it(query, function() {
                     const result = queryInfo.getFormatFromCopyQuery(query);
                     assert.equal(result, 'BINARY');

From 185f708d5b1ac894a5876a04115880611eac3835 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 14:15:55 +0200
Subject: [PATCH 071/133] small style refactor

---
 app/controllers/copy_controller.js | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index dd927194..915a18cc 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -81,7 +81,7 @@ function handleCopyTo (statsClient) {
         res.header("Content-Type", "application/octet-stream");
 
         try {
-            const start_time = Date.now();
+            const startTime = Date.now();
 
             // Open pgsql COPY pipe and stream out to HTTP response
             const pg = new PSQL(res.locals.userDbParams);
@@ -90,12 +90,13 @@ function handleCopyTo (statsClient) {
                     return next(err);
                 }
 
-                const pgstream = client.query(copyTo(sql));
+                const copyToStream = copyTo(sql);
+                const pgstream = client.query(copyToStream);
                 pgstream
                     .on('error', next)
                     .on('data', data => metrics.size += data.length)
                     .on('end', () => {
-                        metrics.time = (Date.now() - start_time) / 1000;
+                        metrics.time = (Date.now() - startTime) / 1000;
                         statsClient.set('copyTo', JSON.stringify(metrics));
                     })
                     .pipe(res);
@@ -122,7 +123,7 @@ function handleCopyFrom () {
         res.locals.copyFromSize = 0;
 
         try {
-            const start_time = Date.now();
+            const startTime = Date.now();
 
             // Connect and run the COPY
             const pg = new PSQL(res.locals.userDbParams);
@@ -133,16 +134,16 @@ function handleCopyFrom () {
 
                 let copyFromStream = copyFrom(sql);
                 const pgstream = client.query(copyFromStream);
-                pgstream.on('error', next);
-                pgstream.on('end', function () {
-                    const end_time = Date.now();
-                    res.body = {
-                        time: (end_time - start_time) / 1000,
-                        total_rows: copyFromStream.rowCount
-                    };
+                pgstream
+                    .on('error', next)
+                    .on('end', function () {
+                        res.body = {
+                            time: (Date.now() - startTime) / 1000,
+                            total_rows: copyFromStream.rowCount
+                        };
 
-                    return next();
-                });
+                        return next();
+                    });
 
                 if (req.get('content-encoding') === 'gzip') {
                     req

From 0bac7a484cdb482b5f23641a67a4fe13b6a266d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 14:16:22 +0200
Subject: [PATCH 072/133] improving query info

---
 app/utils/query_info.js      | 19 +++++++++++++------
 test/unit/query_info.test.js | 18 ++++++++++++++++--
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/app/utils/query_info.js b/app/utils/query_info.js
index b5815063..cc184dff 100644
--- a/app/utils/query_info.js
+++ b/app/utils/query_info.js
@@ -6,14 +6,21 @@ module.exports = {
 
         copyQuery = copyQuery.toUpperCase();
 
-        if (!copyQuery.toUpperCase().startsWith("COPY ")) {
+        if (!copyQuery.startsWith("COPY ")) {
             return false;
         }
-       
-        const regex = /\bFORMAT\s+(\w+)/;
-        const result = regex.exec(copyQuery);
-        if (result && result.length === 2 && COPY_FORMATS.includes(result[1])) {
-            format = result[1];
+        
+        if(copyQuery.includes(' WITH ') && copyQuery.includes('FORMAT ')) {
+            const regex = /\bFORMAT\s+(\w+)/;
+            const result = regex.exec(copyQuery);
+    
+            if (result && result.length === 2) {
+                if (COPY_FORMATS.includes(result[1])) {
+                    format = result[1];
+                } else {
+                    format = false;
+                }
+            }
         }
 
         return format;
diff --git a/test/unit/query_info.test.js b/test/unit/query_info.test.js
index 97202c04..5f889025 100644
--- a/test/unit/query_info.test.js
+++ b/test/unit/query_info.test.js
@@ -2,8 +2,8 @@ const assert = require('assert');
 const queryInfo = require('../../app/utils/query_info');
 
 describe('query info', function () {
-    describe('copy format', function() {
-        describe('csv', function() {
+    describe('copy format', function () {
+        describe('csv', function () {
             const validQueries = [
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT  CSV, DELIMITER ',', HEADER true)",
@@ -45,5 +45,19 @@ describe('query info', function () {
                 });
             });
         });
+
+        describe('should fail', function() {
+            const validQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT ERROR)",
+                "SELECT * from copy_endpoints_test"
+            ];
+
+            validQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.strictEqual(result, false);
+                });
+            });
+        });
     });
 });

From b9474e7fc331a76eead9fb6e9f4b9ad0fcaf5a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 15:29:08 +0200
Subject: [PATCH 073/133] total_rows in copyto metrics

---
 app/controllers/copy_controller.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 915a18cc..c186fc73 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -74,7 +74,8 @@ function handleCopyTo (statsClient) {
         let metrics = {
             size: 0,
             time: null,
-            format: getFormatFromCopyQuery(req.query.sql)
+            format: getFormatFromCopyQuery(req.query.sql),
+            total_rows: null
         };
 
         res.header("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
@@ -97,6 +98,7 @@ function handleCopyTo (statsClient) {
                     .on('data', data => metrics.size += data.length)
                     .on('end', () => {
                         metrics.time = (Date.now() - startTime) / 1000;
+                        metrics.total_rows = copyToStream.rowCount;
                         statsClient.set('copyTo', JSON.stringify(metrics));
                     })
                     .pipe(res);

From e34798546511e23018156a39d2b2d5d39029dfd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 15:42:57 +0200
Subject: [PATCH 074/133] changing query parameter name from sql to q unify
 query validatrion

---
 app/controllers/copy_controller.js | 45 +++++++++++++++---------------
 test/acceptance/copy-endpoints.js  | 12 ++++----
 2 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index c186fc73..33298ad0 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -34,6 +34,7 @@ CopyController.prototype.route = function (app) {
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
+            validateCopyQuery(),
             handleCopyFrom(),
             responseCopyFrom(),
             errorMiddleware()
@@ -48,6 +49,7 @@ CopyController.prototype.route = function (app) {
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
+            validateCopyQuery(),
             handleCopyTo(this.statsClient),
             errorMiddleware()
         ];
@@ -59,22 +61,13 @@ CopyController.prototype.route = function (app) {
 
 function handleCopyTo (statsClient) {
     return function handleCopyToMiddleware (req, res, next) {
-        const { sql } = req.query;
+        const sql = req.query.q;
         const filename = req.query.filename || 'carto-sql-copyto.dmp';
 
-        if (!sql) {
-            throw new Error("Parameter 'sql' is missing");
-        }
-
-        // Only accept SQL that starts with 'COPY'
-        if (!sql.toUpperCase().startsWith("COPY ")) {
-            throw new Error("SQL must start with COPY");
-        }
-
         let metrics = {
             size: 0,
             time: null,
-            format: getFormatFromCopyQuery(req.query.sql),
+            format: getFormatFromCopyQuery(sql),
             total_rows: null
         };
 
@@ -111,16 +104,7 @@ function handleCopyTo (statsClient) {
 
 function handleCopyFrom () {
     return function handleCopyFromMiddleware (req, res, next) {
-        const { sql } = req.query;
-
-        if (!sql) {
-            return next(new Error("Parameter 'sql' is missing, must be in URL or first field in POST"));
-        }
-
-        // Only accept SQL that starts with 'COPY'
-        if (!sql.toUpperCase().startsWith("COPY ")) {
-            return next(new Error("SQL must start with COPY"));
-        }
+        const sql = req.query.q;
 
         res.locals.copyFromSize = 0;
 
@@ -174,7 +158,7 @@ function responseCopyFrom () {
         if (req.profiler) {
             const copyFromMetrics = {
                 size: res.locals.copyFromSize, //bytes
-                format: getFormatFromCopyQuery(req.query.sql),
+                format: getFormatFromCopyQuery(req.query.q),
                 time: res.body.time, //seconds
                 total_rows: res.body.total_rows, 
                 gzip: req.get('content-encoding') === 'gzip'
@@ -188,4 +172,21 @@ function responseCopyFrom () {
     };
 }
 
+function validateCopyQuery () {
+    return function validateCopyQueryMiddleware (req, res, next) {
+        const sql = req.query.q;
+
+        if (!sql) {
+            next(new Error("SQL is missing"));
+        }
+
+        // Only accept SQL that starts with 'COPY'
+        if (!sql.toUpperCase().startsWith("COPY ")) {
+            next(new Error("SQL must start with COPY"));
+        }
+        
+        next();
+    };
+}
+
 module.exports = CopyController;
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 7f58d790..11491437 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -21,7 +21,7 @@ describe('copy-endpoints', function() {
     it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
@@ -49,7 +49,7 @@ describe('copy-endpoints', function() {
     it('should fail with copyfrom endpoint and unexisting table', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
             headers: {host: 'vizzuality.cartodb.com'},
@@ -69,7 +69,7 @@ describe('copy-endpoints', function() {
     it('should fail with copyfrom endpoint and without csv', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'POST'
@@ -96,7 +96,7 @@ describe('copy-endpoints', function() {
             assert.deepEqual(
                 JSON.parse(res.body), 
                 {
-                    error:["Parameter 'sql' is missing, must be in URL or first field in POST"]
+                    error:["SQL is missing"]
                 }
             );
             done();
@@ -106,7 +106,7 @@ describe('copy-endpoints', function() {
     it('should work with copyto endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyto?" + querystring.stringify({
-                sql: 'COPY copy_endpoints_test TO STDOUT',
+                q: 'COPY copy_endpoints_test TO STDOUT',
                 filename: '/tmp/output.dmp'
             }),
             headers: {host: 'vizzuality.cartodb.com'},
@@ -128,7 +128,7 @@ describe('copy-endpoints', function() {
     it('should work with copyfrom and gzip', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
-                sql: "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                q: "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
             }),
             data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv.gz'),
             headers: {

From 30cb88c3f93468372c4d3614c6d4ddaefcff10e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 15:56:04 +0200
Subject: [PATCH 075/133] test for copyto without sql

---
 test/acceptance/copy-endpoints.js | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 11491437..f5458145 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -125,6 +125,25 @@ describe('copy-endpoints', function() {
         });
     });
 
+    it('should fail with copyto endpoint and without sql', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql/copyto?" + querystring.stringify({
+                filename: '/tmp/output.dmp'
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            assert.deepEqual(
+                JSON.parse(res.body), 
+                {
+                    error:["SQL is missing"]
+                }
+            );
+            done();
+        });
+    });
+
     it('should work with copyfrom and gzip', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({

From 6d73d97ae2a8c72e5449fb52c674506b59a32d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 16:02:14 +0200
Subject: [PATCH 076/133] copyFrom metrics to statsd

---
 app/controllers/copy_controller.js | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 33298ad0..6374996d 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -36,7 +36,7 @@ CopyController.prototype.route = function (app) {
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
             handleCopyFrom(),
-            responseCopyFrom(),
+            responseCopyFrom(this.statsClient),
             errorMiddleware()
         ];
     };
@@ -149,25 +149,22 @@ function handleCopyFrom () {
     };
 }
 
-function responseCopyFrom () {
+function responseCopyFrom (statsClient) {
     return function responseCopyFromMiddleware (req, res, next) {
         if (!res.body || !res.body.total_rows) {
             return next(new Error("No rows copied"));
         }
 
-        if (req.profiler) {
-            const copyFromMetrics = {
-                size: res.locals.copyFromSize, //bytes
-                format: getFormatFromCopyQuery(req.query.q),
-                time: res.body.time, //seconds
-                total_rows: res.body.total_rows, 
-                gzip: req.get('content-encoding') === 'gzip'
-            };
-
-            req.profiler.add({ copyFrom: copyFromMetrics });
-            res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
-        }
+        const metrics = {
+            size: res.locals.copyFromSize, //bytes
+            format: getFormatFromCopyQuery(req.query.q),
+            time: res.body.time, //seconds
+            total_rows: res.body.total_rows, 
+            gzip: req.get('content-encoding') === 'gzip'
+        };
 
+        statsClient.set('copyFrom', JSON.stringify(metrics));
+        
         res.send(res.body);
     };
 }
@@ -184,7 +181,7 @@ function validateCopyQuery () {
         if (!sql.toUpperCase().startsWith("COPY ")) {
             next(new Error("SQL must start with COPY"));
         }
-        
+
         next();
     };
 }

From bfeea58268dd05ae71058a6e43818d0ae9285bf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 22 May 2018 16:07:04 +0200
Subject: [PATCH 077/133] copyFrom metrics to statsd tests

---
 test/acceptance/copy-endpoints.js | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index f5458145..847d741c 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -32,16 +32,6 @@ describe('copy-endpoints', function() {
             assert.equal(!!response.time, true);
             assert.strictEqual(response.total_rows, 6);
 
-            assert.ok(res.headers['x-sqlapi-profiler']);
-            const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
-            assert.ok(headers.copyFrom);
-            const metrics = headers.copyFrom;
-            assert.equal(metrics.size, 57);
-            assert.equal(metrics.format, 'CSV');
-            assert.equal(metrics.time, response.time);
-            assert.equal(metrics.total_rows, response.total_rows);
-            assert.equal(metrics.gzip, false);
-
             done();
         });
     });
@@ -161,16 +151,6 @@ describe('copy-endpoints', function() {
             assert.equal(!!response.time, true);
             assert.strictEqual(response.total_rows, 6);
 
-            assert.ok(res.headers['x-sqlapi-profiler']);
-            const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
-            assert.ok(headers.copyFrom);
-            const metrics = headers.copyFrom;
-            assert.equal(metrics.size, 57);
-            assert.equal(metrics.format, 'CSV');
-            assert.equal(metrics.time, response.time);
-            assert.equal(metrics.total_rows, response.total_rows);
-            assert.equal(metrics.gzip, true);
-
             done();
         });
     });

From f77707f6dab6aca6f3d4424f470d42696fbc3bbd Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Tue, 22 May 2018 10:48:11 -0400
Subject: [PATCH 078/133] Fix reponse copy from

---
 app/controllers/copy_controller.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 2ed047be..ce3296b0 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -36,7 +36,7 @@ CopyController.prototype.route = function (app) {
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
             handleCopyFrom(),
-            responseCopyFrom(),
+            responseCopyFrom(this.statsClient),
             errorMiddleware()
         ];
     };

From 037dc00c0f86988795b7637d98c2e86bcaf7b20a Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Tue, 22 May 2018 13:53:33 -0400
Subject: [PATCH 079/133] Remove junk from merge

---
 doc/copy_queries.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 41be7d15..7b4230c5 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -159,7 +159,6 @@ The SQL needs to be URL-encoded before being embedded in the CURL command, so th
 
 The Python to "copy to" is very simple, because the HTTP call is a simple get. The only complexity in this example is at the end, where the result is streamed back block-by-block, to avoid pulling the entire download into memory before writing to file.
 
-<<<<<<< HEAD
 ```python
 import requests
 import re

From 8dd1d5babf1ade4a898287b640affa2d043c6dcd Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Tue, 22 May 2018 15:24:19 -0400
Subject: [PATCH 080/133] Try and quiet multipart query

---
 test/acceptance/query-multipart.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/acceptance/query-multipart.js b/test/acceptance/query-multipart.js
index cb88870e..27f4c3c6 100644
--- a/test/acceptance/query-multipart.js
+++ b/test/acceptance/query-multipart.js
@@ -15,7 +15,7 @@ describe('query-multipart', function() {
         },{}, function(err, res) {
             assert.ifError(err);
             const response = JSON.parse(res.body);
-            assert.equal(!!response.time, true);
+            assert.equal(typeof(response.time) !== 'undefined', true);
             assert.strictEqual(response.total_rows, 1);
             assert.deepStrictEqual(response.rows, [{n:2}]);
             done();

From 5ba7dca79c36f6e99bbeedc553cd0f0ab24f7f77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Wed, 23 May 2018 10:30:37 +0200
Subject: [PATCH 081/133] copyfrom metrics to kibana

---
 app/controllers/copy_controller.js | 23 +++++++++++++----------
 test/acceptance/copy-endpoints.js  | 22 +++++++++++++++++++++-
 2 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index ce3296b0..22ff2c39 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -36,7 +36,7 @@ CopyController.prototype.route = function (app) {
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
             handleCopyFrom(),
-            responseCopyFrom(this.statsClient),
+            responseCopyFrom(),
             errorMiddleware()
         ];
     };
@@ -150,21 +150,24 @@ function handleCopyFrom () {
     };
 }
 
-function responseCopyFrom (statsClient) {
+function responseCopyFrom () {
     return function responseCopyFromMiddleware (req, res, next) {
         if (!res.body || !res.body.total_rows) {
             return next(new Error("No rows copied"));
         }
 
-        const metrics = {
-            size: res.locals.copyFromSize, //bytes
-            format: getFormatFromCopyQuery(req.query.q),
-            time: res.body.time, //seconds
-            total_rows: res.body.total_rows, 
-            gzip: req.get('content-encoding') === 'gzip'
-        };
+        if (req.profiler) {
+            const metrics = {
+                size: res.locals.copyFromSize, //bytes
+                format: getFormatFromCopyQuery(req.query.q),
+                time: res.body.time, //seconds
+                total_rows: res.body.total_rows, 
+                gzip: req.get('content-encoding') === 'gzip'
+            };
 
-        statsClient.set('copyFrom', JSON.stringify(metrics));
+            req.profiler.add({ copyFrom: metrics });	
+            res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
+        }
         
         res.send(res.body);
     };
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index eceab8c1..3108e867 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -32,6 +32,16 @@ describe('copy-endpoints', function() {
             assert.equal(!!response.time, true);
             assert.strictEqual(response.total_rows, 6);
 
+            assert.ok(res.headers['x-sqlapi-profiler']);
+            const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
+            assert.ok(headers.copyFrom);
+            const metrics = headers.copyFrom;
+            assert.equal(metrics.size, 57);
+            assert.equal(metrics.format, 'CSV');
+            assert.equal(metrics.time, response.time);
+            assert.equal(metrics.total_rows, response.total_rows);
+            assert.equal(metrics.gzip, false);
+
             done();
         });
     });
@@ -150,7 +160,17 @@ describe('copy-endpoints', function() {
             const response = JSON.parse(res.body);
             assert.equal(!!response.time, true);
             assert.strictEqual(response.total_rows, 6);
-
+            
+            assert.ok(res.headers['x-sqlapi-profiler']);
+            const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
+            assert.ok(headers.copyFrom);
+            const metrics = headers.copyFrom;
+            assert.equal(metrics.size, 57);
+            assert.equal(metrics.format, 'CSV');
+            assert.equal(metrics.time, response.time);
+            assert.equal(metrics.total_rows, response.total_rows);
+            assert.equal(metrics.gzip, true);
+            
             done();
         });
     });

From fdc542f7b56e5dc3a26f98855c1e96fa2f45c354 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Wed, 23 May 2018 17:24:48 +0200
Subject: [PATCH 082/133] BunyanLogger

---
 app/services/bunyanLogger.js | 33 +++++++++++++++++++++++++++++++++
 batch/batch-logger.js        | 34 ++++++++++++----------------------
 batch/index.js               |  2 +-
 3 files changed, 46 insertions(+), 23 deletions(-)
 create mode 100644 app/services/bunyanLogger.js

diff --git a/app/services/bunyanLogger.js b/app/services/bunyanLogger.js
new file mode 100644
index 00000000..a0ab8920
--- /dev/null
+++ b/app/services/bunyanLogger.js
@@ -0,0 +1,33 @@
+'use strict';
+
+const bunyan = require('bunyan');
+
+class BunyanLogger {
+    constructor (path, name) {
+        const stream = {
+            level: process.env.NODE_ENV === 'test' ? 'fatal' : 'info'
+        };
+
+        if (path) {
+            stream.path = path;
+        } else {
+            stream.stream = process.stdout;
+        }
+
+        this.path = path;
+        this.logger = bunyan.createLogger({
+            name,
+            streams: [stream]
+        });
+    }
+
+    info (log, message) {
+        this.logger.info(log, message);
+    }
+
+    warn (log, message) {
+        this.logger.warn(log, message);
+    }
+}
+ 
+module.exports = BunyanLogger;
diff --git a/batch/batch-logger.js b/batch/batch-logger.js
index 05dc54f8..b424a7cc 100644
--- a/batch/batch-logger.js
+++ b/batch/batch-logger.js
@@ -1,29 +1,19 @@
 'use strict';
 
-var bunyan = require('bunyan');
+const BunyanLogger = require('../app/services/bunyanLogger');
 
-function BatchLogger (path) {
-    var stream = {
-        level: process.env.NODE_ENV === 'test' ? 'fatal' : 'info'
-    };
-    if (path) {
-        stream.path = path;
-    } else {
-        stream.stream = process.stdout;
+class BatchLogger extends BunyanLogger {
+    constructor (path, name) {
+        super(path, name);
+    }
+
+    log (job) {
+        return job.log(this.logger);
+    }
+
+    reopenFileStreams () {
+        this.logger.reopenFileStreams();
     }
-    this.path = path;
-    this.logger = bunyan.createLogger({
-        name: 'batch-queries',
-        streams: [stream]
-    });
 }
 
 module.exports = BatchLogger;
-
-BatchLogger.prototype.log = function (job) {
-    return job.log(this.logger);
-};
-
-BatchLogger.prototype.reopenFileStreams = function () {
-    this.logger.reopenFileStreams();
-};
diff --git a/batch/index.js b/batch/index.js
index 2c0e820b..260efdc9 100644
--- a/batch/index.js
+++ b/batch/index.js
@@ -24,7 +24,7 @@ module.exports = function batchFactory (metadataBackend, redisPool, name, statsd
     var jobCanceller = new JobCanceller(userDatabaseMetadataService);
     var jobService = new JobService(jobBackend, jobCanceller);
     var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient);
-    var logger = new BatchLogger(loggerPath);
+    var logger = new BatchLogger(loggerPath, 'batch-queries');
 
     return new Batch(
         name,

From b2a36eb556a173eb65fb003117d818f490d45ae7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Wed, 23 May 2018 17:25:46 +0200
Subject: [PATCH 083/133] copy metrics with BunyanLogger

---
 app/controllers/copy_controller.js         | 36 +++++++++++++---------
 app/server.js                              |  3 +-
 config/environments/development.js.example |  1 +
 config/environments/production.js.example  |  1 +
 config/environments/staging.js.example     |  1 +
 config/environments/test.js.example        |  1 +
 6 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 22ff2c39..3381590b 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -9,6 +9,7 @@ const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 const { getFormatFromCopyQuery } = require('../utils/query_info');
+const BunyanLogger = require('../services/bunyanLogger');
 
 const zlib = require('zlib');
 const PSQL = require('cartodb-psql');
@@ -21,6 +22,8 @@ function CopyController(metadataBackend, userDatabaseService, userLimitsService,
     this.userDatabaseService = userDatabaseService;
     this.userLimitsService = userLimitsService;
     this.statsClient = statsClient;
+
+    this.logger = new BunyanLogger(global.settings.dataIngestionFilename, 'data-ingestion');
 }
 
 CopyController.prototype.route = function (app) {
@@ -36,7 +39,7 @@ CopyController.prototype.route = function (app) {
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
             handleCopyFrom(),
-            responseCopyFrom(),
+            responseCopyFrom(this.logger),
             errorMiddleware()
         ];
     };
@@ -50,7 +53,7 @@ CopyController.prototype.route = function (app) {
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
-            handleCopyTo(this.statsClient),
+            handleCopyTo(this.logger),
             errorMiddleware()
         ];
     };
@@ -60,12 +63,13 @@ CopyController.prototype.route = function (app) {
 };
 
 
-function handleCopyTo (statsClient) {
+function handleCopyTo (logger) {
     return function handleCopyToMiddleware (req, res, next) {
         const sql = req.query.q;
         const filename = req.query.filename || 'carto-sql-copyto.dmp';
 
         let metrics = {
+            type: 'copyto',
             size: 0,
             time: null,
             format: getFormatFromCopyQuery(sql),
@@ -93,7 +97,7 @@ function handleCopyTo (statsClient) {
                     .on('end', () => {
                         metrics.time = (Date.now() - startTime) / 1000;
                         metrics.total_rows = copyToStream.rowCount;
-                        statsClient.set('copyTo', JSON.stringify(metrics));
+                        logger.info(metrics);
                     })
                     .pipe(res);
             });
@@ -150,23 +154,27 @@ function handleCopyFrom () {
     };
 }
 
-function responseCopyFrom () {
+function responseCopyFrom (logger) {
     return function responseCopyFromMiddleware (req, res, next) {
         if (!res.body || !res.body.total_rows) {
             return next(new Error("No rows copied"));
         }
 
-        if (req.profiler) {
-            const metrics = {
-                size: res.locals.copyFromSize, //bytes
-                format: getFormatFromCopyQuery(req.query.q),
-                time: res.body.time, //seconds
-                total_rows: res.body.total_rows, 
-                gzip: req.get('content-encoding') === 'gzip'
-            };
+        const metrics = {
+            type: 'copyfrom',
+            size: res.locals.copyFromSize, //bytes
+            format: getFormatFromCopyQuery(req.query.q),
+            time: res.body.time, //seconds
+            total_rows: res.body.total_rows, 
+            gzip: req.get('content-encoding') === 'gzip'
+        };
+        
+        logger.info(metrics);
 
+        // TODO: remove when data-ingestion log works
+        if (req.profiler) {
             req.profiler.add({ copyFrom: metrics });	
-            res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
+            res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());    
         }
         
         res.send(res.body);
diff --git a/app/server.js b/app/server.js
index c005346c..bbd079f6 100644
--- a/app/server.js
+++ b/app/server.js
@@ -178,8 +178,7 @@ function App(statsClient) {
     var copyController = new CopyController(
         metadataBackend, 
         userDatabaseService, 
-        userLimitsService,
-        statsClient
+        userLimitsService
     );
     copyController.route(app);
     
diff --git a/config/environments/development.js.example b/config/environments/development.js.example
index adc7c001..22f70fde 100644
--- a/config/environments/development.js.example
+++ b/config/environments/development.js.example
@@ -122,3 +122,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
+module.exports.dataIngestionFilename = 'logs/data-ingestion.log';
diff --git a/config/environments/production.js.example b/config/environments/production.js.example
index 0ee13ad2..875b9aa8 100644
--- a/config/environments/production.js.example
+++ b/config/environments/production.js.example
@@ -126,3 +126,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
+module.exports.dataIngestionFilename = 'logs/data-ingestion.log';
diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example
index 99b336bd..38ca4ea6 100644
--- a/config/environments/staging.js.example
+++ b/config/environments/staging.js.example
@@ -123,3 +123,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
+module.exports.dataIngestionFilename = 'logs/data-ingestion.log';
diff --git a/config/environments/test.js.example b/config/environments/test.js.example
index e41f1d5b..e5c5d67d 100644
--- a/config/environments/test.js.example
+++ b/config/environments/test.js.example
@@ -123,3 +123,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
+module.exports.dataIngestionFilename = 'logs/data-ingestion.log';

From fca6ee82323f5df8a66909b12e11bbb17303b9e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Wed, 23 May 2018 17:32:44 +0200
Subject: [PATCH 084/133] changing name of dataIngestionLogPath

---
 app/controllers/copy_controller.js         | 2 +-
 config/environments/development.js.example | 2 +-
 config/environments/production.js.example  | 2 +-
 config/environments/staging.js.example     | 2 +-
 config/environments/test.js.example        | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 3381590b..d7bb5ed2 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -23,7 +23,7 @@ function CopyController(metadataBackend, userDatabaseService, userLimitsService,
     this.userLimitsService = userLimitsService;
     this.statsClient = statsClient;
 
-    this.logger = new BunyanLogger(global.settings.dataIngestionFilename, 'data-ingestion');
+    this.logger = new BunyanLogger(global.settings.dataIngestionLogPath, 'data-ingestion');
 }
 
 CopyController.prototype.route = function (app) {
diff --git a/config/environments/development.js.example b/config/environments/development.js.example
index 22f70fde..abaa2b94 100644
--- a/config/environments/development.js.example
+++ b/config/environments/development.js.example
@@ -122,4 +122,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
-module.exports.dataIngestionFilename = 'logs/data-ingestion.log';
+module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
diff --git a/config/environments/production.js.example b/config/environments/production.js.example
index 875b9aa8..45e00732 100644
--- a/config/environments/production.js.example
+++ b/config/environments/production.js.example
@@ -126,4 +126,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
-module.exports.dataIngestionFilename = 'logs/data-ingestion.log';
+module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example
index 38ca4ea6..2c5929a6 100644
--- a/config/environments/staging.js.example
+++ b/config/environments/staging.js.example
@@ -123,4 +123,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
-module.exports.dataIngestionFilename = 'logs/data-ingestion.log';
+module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
diff --git a/config/environments/test.js.example b/config/environments/test.js.example
index e5c5d67d..42255fb2 100644
--- a/config/environments/test.js.example
+++ b/config/environments/test.js.example
@@ -123,4 +123,4 @@ module.exports.ratelimits = {
 }
 
 module.exports.validatePGEntitiesAccess = false;
-module.exports.dataIngestionFilename = 'logs/data-ingestion.log';
+module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';

From 3d8f45afd8661005e12be0e9e9d900083e10fbab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Thu, 24 May 2018 19:08:35 +0200
Subject: [PATCH 085/133] going red

---
 test/acceptance/copy-endpoints.js | 83 +++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 3108e867..0a847827 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -176,3 +176,86 @@ describe('copy-endpoints', function() {
     });
 
 });
+
+
+describe('copy-endpoints timeout', function() {
+    it('should fail with copyfrom and timeout', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=set statement_timeout = 10',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },
+        function(err, res) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },
+            {
+                status: 429,
+                headers: { 'Content-Type': 'application/json; charset=utf-8' }
+            },
+            function(err, res) {
+                assert.ifError(err);
+                assert.deepEqual(JSON.parse(res.body), {
+                    error: [
+                        'You are over platform\'s limits. Please contact us to know more details'
+                    ],
+                    context: 'limit',
+                    detail: 'datasource'
+                });
+    
+                
+                assert.response(server, {
+                    url: "/api/v1/sql?q=set statement_timeout = 2000",
+                    headers: {host: 'vizzuality.cartodb.com'},
+                    method: 'GET'
+                },
+                function(err, res) {
+                    done();
+                });
+            });
+        });
+    });
+
+    it('should fail with copyto and timeout', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=set statement_timeout = 1',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },
+        function(err, res) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyto?" + querystring.stringify({
+                    q: 'COPY populated_places_simple_reduced TO STDOUT',
+                    filename: '/tmp/output.dmp'
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                // assert.deepEqual(JSON.parse(res.body), {
+                //     error: [
+                //         'You are over platform\'s limits. Please contact us to know more details'
+                //     ],
+                //     context: 'limit',
+                //     detail: 'datasource'
+                // });
+                console.log(res.body);
+    
+                
+                assert.response(server, {
+                    url: "/api/v1/sql?q=set statement_timeout = 2000",
+                    headers: {host: 'vizzuality.cartodb.com'},
+                    method: 'GET'
+                },
+                function(err, res) {
+                    done();
+                });
+            });
+        });
+    });
+})

From e2bba967f996498305b5d88e37b079fe182d9f25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Thu, 24 May 2018 19:48:24 +0200
Subject: [PATCH 086/133] handling copyto errors

---
 app/controllers/copy_controller.js |  8 ++++++-
 test/acceptance/copy-endpoints.js  | 34 +++++++++++++-----------------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index d7bb5ed2..2203713a 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -10,6 +10,7 @@ const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 const { getFormatFromCopyQuery } = require('../utils/query_info');
 const BunyanLogger = require('../services/bunyanLogger');
+const errorHandlerFactory = require('../services/error_handler_factory');
 
 const zlib = require('zlib');
 const PSQL = require('cartodb-psql');
@@ -92,7 +93,12 @@ function handleCopyTo (logger) {
                 const copyToStream = copyTo(sql);
                 const pgstream = client.query(copyToStream);
                 pgstream
-                    .on('error', next)
+                    .on('error', err => {
+                        pgstream.unpipe(res);
+                        const errorHandler = errorHandlerFactory(err);
+                        res.write(JSON.stringify(errorHandler.getResponse()));
+                        res.end();
+                    })
                     .on('data', data => metrics.size += data.length)
                     .on('end', () => {
                         metrics.time = (Date.now() - startTime) / 1000;
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 0a847827..98566ffe 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -185,7 +185,8 @@ describe('copy-endpoints timeout', function() {
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'GET'
         },
-        function(err, res) {
+        function(err) {
+            assert.ifError(err);
             assert.response(server, {
                 url: "/api/v1/sql/copyfrom?" + querystring.stringify({
                     q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
@@ -214,20 +215,19 @@ describe('copy-endpoints timeout', function() {
                     headers: {host: 'vizzuality.cartodb.com'},
                     method: 'GET'
                 },
-                function(err, res) {
-                    done();
-                });
+                done);
             });
         });
     });
 
     it('should fail with copyto and timeout', function(done){
         assert.response(server, {
-            url: '/api/v1/sql?q=set statement_timeout = 1',
+            url: '/api/v1/sql?q=set statement_timeout = 20',
             headers: {host: 'vizzuality.cartodb.com'},
             method: 'GET'
         },
-        function(err, res) {
+        function(err) {
+            assert.ifError(err);
             assert.response(server, {
                 url: "/api/v1/sql/copyto?" + querystring.stringify({
                     q: 'COPY populated_places_simple_reduced TO STDOUT',
@@ -237,25 +237,21 @@ describe('copy-endpoints timeout', function() {
                 method: 'GET'
             },{}, function(err, res) {
                 assert.ifError(err);
-                // assert.deepEqual(JSON.parse(res.body), {
-                //     error: [
-                //         'You are over platform\'s limits. Please contact us to know more details'
-                //     ],
-                //     context: 'limit',
-                //     detail: 'datasource'
-                // });
-                console.log(res.body);
-    
+                const error = {
+                    error:["You are over platform's limits. Please contact us to know more details"],
+                    context:"limit",
+                    detail:"datasource"
+                };
+                const expectedError = res.body.substring(res.body.length - JSON.stringify(error).length);
+                assert.deepEqual(JSON.parse(expectedError), error);
                 
                 assert.response(server, {
                     url: "/api/v1/sql?q=set statement_timeout = 2000",
                     headers: {host: 'vizzuality.cartodb.com'},
                     method: 'GET'
                 },
-                function(err, res) {
-                    done();
-                });
+                done);
             });
         });
     });
-})
+});

From fe3bd4fd37b6d1d39a9ca78abbc7a1c6f6991b1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 14:33:54 +0200
Subject: [PATCH 087/133] handling errors from request in COPYfrom

---
 app/controllers/copy_controller.js | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 2203713a..7794a94d 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -132,8 +132,11 @@ function handleCopyFrom () {
                 let copyFromStream = copyFrom(sql);
                 const pgstream = client.query(copyFromStream);
                 pgstream
-                    .on('error', next)
-                    .on('end', function () {
+                    .on('error', err => {
+                        req.unpipe(pgstream);
+                        return next(err);
+                    })
+                    .on('end', function () {                        
                         res.body = {
                             time: (Date.now() - startTime) / 1000,
                             total_rows: copyFromStream.rowCount
@@ -145,10 +148,31 @@ function handleCopyFrom () {
                 if (req.get('content-encoding') === 'gzip') {
                     req
                         .pipe(zlib.createGunzip())
+                        .on('error', err => {
+                            req.unpipe(pgstream);
+                            pgstream.end();
+                            return next(err);
+                        })
+                        .on('close', err => {
+                            req.unpipe(pgstream);
+                            pgstream.end();
+                            return next(new Error('Connection closed by client'));
+                        })
                         .on('data', data => res.locals.copyFromSize += data.length)
                         .pipe(pgstream);
                 } else {
                     req
+                        .on('error', err => {
+                            req.unpipe(pgstream);
+                            pgstream.end();
+                            return next(err);
+                        })
+                        .on('close', err => {
+                            req.unpipe(pgstream);
+                            // pgstream.write(Buffer.from([0x66]))
+                            pgstream.end();
+                            return next(new Error('Connection closed by client'));
+                        })
                         .on('data', data => res.locals.copyFromSize += data.length)
                         .pipe(pgstream);
                 }

From 5bed04a38ce15f73185414daa9285023018705fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 15:34:30 +0200
Subject: [PATCH 088/133] fix gzip problem with req close event

---
 app/controllers/copy_controller.js | 50 ++++++++++++------------------
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 7794a94d..318533d9 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -136,7 +136,7 @@ function handleCopyFrom () {
                         req.unpipe(pgstream);
                         return next(err);
                     })
-                    .on('end', function () {                        
+                    .on('end', function () {
                         res.body = {
                             time: (Date.now() - startTime) / 1000,
                             total_rows: copyFromStream.rowCount
@@ -145,37 +145,27 @@ function handleCopyFrom () {
                         return next();
                     });
 
+                let request = req;
                 if (req.get('content-encoding') === 'gzip') {
-                    req
-                        .pipe(zlib.createGunzip())
-                        .on('error', err => {
-                            req.unpipe(pgstream);
-                            pgstream.end();
-                            return next(err);
-                        })
-                        .on('close', err => {
-                            req.unpipe(pgstream);
-                            pgstream.end();
-                            return next(new Error('Connection closed by client'));
-                        })
-                        .on('data', data => res.locals.copyFromSize += data.length)
-                        .pipe(pgstream);
-                } else {
-                    req
-                        .on('error', err => {
-                            req.unpipe(pgstream);
-                            pgstream.end();
-                            return next(err);
-                        })
-                        .on('close', err => {
-                            req.unpipe(pgstream);
-                            // pgstream.write(Buffer.from([0x66]))
-                            pgstream.end();
-                            return next(new Error('Connection closed by client'));
-                        })
-                        .on('data', data => res.locals.copyFromSize += data.length)
-                        .pipe(pgstream);
+                    request = req.pipe(zlib.createGunzip());
                 }
+
+                request
+                    .on('error', err => {
+                        req.unpipe(pgstream);
+                        pgstream.end();
+                        return next(err);
+                    })
+                    .on('close', () => {
+                        if (!request.ended) {
+                            req.unpipe(pgstream);
+                            pgstream.end();
+                            return next(new Error('Connection closed by client'));
+                        }
+                    })
+                    .on('data', data => res.locals.copyFromSize += data.length)
+                    .on('end', () => request.ended = true)
+                    .pipe(pgstream);
             });
 
         } catch (err) {

From 9b0b92fb6b7bf2149660a04428dd0bf28e14435a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 15:46:12 +0200
Subject: [PATCH 089/133] removing try catch

---
 app/controllers/copy_controller.js | 136 +++++++++++++----------------
 1 file changed, 62 insertions(+), 74 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 318533d9..a9bed62e 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -80,97 +80,85 @@ function handleCopyTo (logger) {
         res.header("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
         res.header("Content-Type", "application/octet-stream");
 
-        try {
-            const startTime = Date.now();
+        const startTime = Date.now();
 
-            // Open pgsql COPY pipe and stream out to HTTP response
-            const pg = new PSQL(res.locals.userDbParams);
-            pg.connect(function (err, client) {
-                if (err) {
-                    return next(err);
-                }
+        // Open pgsql COPY pipe and stream out to HTTP response
+        const pg = new PSQL(res.locals.userDbParams);
+        pg.connect(function (err, client) {
+            if (err) {
+                return next(err);
+            }
 
-                const copyToStream = copyTo(sql);
-                const pgstream = client.query(copyToStream);
-                pgstream
-                    .on('error', err => {
-                        pgstream.unpipe(res);
-                        const errorHandler = errorHandlerFactory(err);
-                        res.write(JSON.stringify(errorHandler.getResponse()));
-                        res.end();
-                    })
-                    .on('data', data => metrics.size += data.length)
-                    .on('end', () => {
-                        metrics.time = (Date.now() - startTime) / 1000;
-                        metrics.total_rows = copyToStream.rowCount;
-                        logger.info(metrics);
-                    })
-                    .pipe(res);
-            });
-        } catch (err) {
-            next(err);
-        }
+            const copyToStream = copyTo(sql);
+            const pgstream = client.query(copyToStream);
+            pgstream
+                .on('error', err => {
+                    pgstream.unpipe(res);
+                    const errorHandler = errorHandlerFactory(err);
+                    res.write(JSON.stringify(errorHandler.getResponse()));
+                    res.end();
+                })
+                .on('data', data => metrics.size += data.length)
+                .on('end', () => {
+                    metrics.time = (Date.now() - startTime) / 1000;
+                    metrics.total_rows = copyToStream.rowCount;
+                    logger.info(metrics);
+                })
+                .pipe(res);
+        });
     };
 }
 
 function handleCopyFrom () {
     return function handleCopyFromMiddleware (req, res, next) {
         const sql = req.query.q;
-
+        const startTime = Date.now();
         res.locals.copyFromSize = 0;
 
-        try {
-            const startTime = Date.now();
+        const pg = new PSQL(res.locals.userDbParams);
+        pg.connect(function (err, client) {
+            if (err) {
+                return next(err);
+            }
 
-            // Connect and run the COPY
-            const pg = new PSQL(res.locals.userDbParams);
-            pg.connect(function (err, client) {
-                if (err) {
+            let copyFromStream = copyFrom(sql);
+            const pgstream = client.query(copyFromStream);
+            pgstream
+                .on('error', err => {
+                    req.unpipe(pgstream);
                     return next(err);
-                }
+                })
+                .on('end', function () {
+                    res.body = {
+                        time: (Date.now() - startTime) / 1000,
+                        total_rows: copyFromStream.rowCount
+                    };
 
-                let copyFromStream = copyFrom(sql);
-                const pgstream = client.query(copyFromStream);
-                pgstream
-                    .on('error', err => {
-                        req.unpipe(pgstream);
-                        return next(err);
-                    })
-                    .on('end', function () {
-                        res.body = {
-                            time: (Date.now() - startTime) / 1000,
-                            total_rows: copyFromStream.rowCount
-                        };
+                    return next();
+                });
 
-                        return next();
-                    });
+            let request = req;
+            if (req.get('content-encoding') === 'gzip') {
+                request = req.pipe(zlib.createGunzip());
+            }
 
-                let request = req;
-                if (req.get('content-encoding') === 'gzip') {
-                    request = req.pipe(zlib.createGunzip());
-                }
-
-                request
-                    .on('error', err => {
+            request
+                .on('error', err => {
+                    req.unpipe(pgstream);
+                    pgstream.end();
+                    return next(err);
+                })
+                .on('close', () => {
+                    if (!request.ended) {
                         req.unpipe(pgstream);
                         pgstream.end();
-                        return next(err);
-                    })
-                    .on('close', () => {
-                        if (!request.ended) {
-                            req.unpipe(pgstream);
-                            pgstream.end();
-                            return next(new Error('Connection closed by client'));
-                        }
-                    })
-                    .on('data', data => res.locals.copyFromSize += data.length)
-                    .on('end', () => request.ended = true)
-                    .pipe(pgstream);
-            });
-
-        } catch (err) {
-            next(err);
-        }
+                        return next(new Error('Connection closed by client'));
+                    }
+                })
+                .on('data', data => res.locals.copyFromSize += data.length)
+                .on('end', () => request.ended = true)
+                .pipe(pgstream);
+        });
     };
 }
 

From 8a2c7775772cddff2770ac21a06bb5ea0dc1902b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 16:00:27 +0200
Subject: [PATCH 090/133] details

---
 app/controllers/copy_controller.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index a9bed62e..3b088b4a 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -82,7 +82,6 @@ function handleCopyTo (logger) {
 
         const startTime = Date.now();
 
-        // Open pgsql COPY pipe and stream out to HTTP response
         const pg = new PSQL(res.locals.userDbParams);
         pg.connect(function (err, client) {
             if (err) {
@@ -112,8 +111,9 @@ function handleCopyTo (logger) {
 function handleCopyFrom () {
     return function handleCopyFromMiddleware (req, res, next) {
         const sql = req.query.q;
-        const startTime = Date.now();
         res.locals.copyFromSize = 0;
+        
+        const startTime = Date.now();
 
         const pg = new PSQL(res.locals.userDbParams);
         pg.connect(function (err, client) {

From fd70673d88b8888dfaede187a63dcf4c4513871f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 16:46:25 +0200
Subject: [PATCH 091/133] more details

---
 app/controllers/copy_controller.js | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 3b088b4a..f805eb8b 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -137,26 +137,27 @@ function handleCopyFrom () {
                     return next();
                 });
 
-            let request = req;
+            let requestEnded = false;
+
             if (req.get('content-encoding') === 'gzip') {
-                request = req.pipe(zlib.createGunzip());
+                req = req.pipe(zlib.createGunzip());
             }
 
-            request
+            req
                 .on('error', err => {
                     req.unpipe(pgstream);
                     pgstream.end();
                     return next(err);
                 })
                 .on('close', () => {
-                    if (!request.ended) {
+                    if (!requestEnded) {
                         req.unpipe(pgstream);
                         pgstream.end();
                         return next(new Error('Connection closed by client'));
                     }
                 })
                 .on('data', data => res.locals.copyFromSize += data.length)
-                .on('end', () => request.ended = true)
+                .on('end', () => requestEnded = true)
                 .pipe(pgstream);
         });
     };

From 6c3f9cf1d3deb6483251ccb672ba1f1f2aa26f54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 17:42:30 +0200
Subject: [PATCH 092/133] simplify controllers

---
 app/controllers/copy_controller.js | 152 +++++++++--------------------
 app/services/copy_command.js       | 105 ++++++++++++++++++++
 2 files changed, 150 insertions(+), 107 deletions(-)
 create mode 100644 app/services/copy_command.js

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index f805eb8b..e133d49e 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -8,15 +8,9 @@ const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
 const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
-const { getFormatFromCopyQuery } = require('../utils/query_info');
 const BunyanLogger = require('../services/bunyanLogger');
 const errorHandlerFactory = require('../services/error_handler_factory');
-
-const zlib = require('zlib');
-const PSQL = require('cartodb-psql');
-const copyTo = require('pg-copy-streams').to;
-const copyFrom = require('pg-copy-streams').from;
-
+const copyCommand = require('../services/copy_command');
 
 function CopyController(metadataBackend, userDatabaseService, userLimitsService, statsClient) {
     this.metadataBackend = metadataBackend;
@@ -39,8 +33,8 @@ CopyController.prototype.route = function (app) {
             connectionParamsMiddleware(this.userDatabaseService),
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
-            handleCopyFrom(),
-            responseCopyFrom(this.logger),
+            handleCopyFrom(this.logger),
+            responseCopyFrom(),
             errorMiddleware()
         ];
     };
@@ -66,125 +60,70 @@ CopyController.prototype.route = function (app) {
 
 function handleCopyTo (logger) {
     return function handleCopyToMiddleware (req, res, next) {
-        const sql = req.query.q;
         const filename = req.query.filename || 'carto-sql-copyto.dmp';
 
-        let metrics = {
-            type: 'copyto',
-            size: 0,
-            time: null,
-            format: getFormatFromCopyQuery(sql),
-            total_rows: null
-        };
-
         res.header("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
         res.header("Content-Type", "application/octet-stream");
 
-        const startTime = Date.now();
-
-        const pg = new PSQL(res.locals.userDbParams);
-        pg.connect(function (err, client) {
+        copyCommand.streamCopyTo(
+            res, 
+            req.query.q, 
+            res.locals.userDbParams, 
+            function(err, metrics) {
             if (err) {
-                return next(err);
-            }
-
-            const copyToStream = copyTo(sql);
-            const pgstream = client.query(copyToStream);
-            pgstream
-                .on('error', err => {
-                    pgstream.unpipe(res);
+                if (res.headersSent) {
                     const errorHandler = errorHandlerFactory(err);
                     res.write(JSON.stringify(errorHandler.getResponse()));
                     res.end();
-                })
-                .on('data', data => metrics.size += data.length)
-                .on('end', () => {
-                    metrics.time = (Date.now() - startTime) / 1000;
-                    metrics.total_rows = copyToStream.rowCount;
-                    logger.info(metrics);
-                })
-                .pipe(res);
+                } else {
+                    return next(err);
+                }
+            } else {
+                logger.info(metrics);
+            }
         });
     };
 }
 
-function handleCopyFrom () {
+function streamCopyFrom (logger) {
     return function handleCopyFromMiddleware (req, res, next) {
-        const sql = req.query.q;
-        res.locals.copyFromSize = 0;
-        
-        const startTime = Date.now();
-
-        const pg = new PSQL(res.locals.userDbParams);
-        pg.connect(function (err, client) {
-            if (err) {
-                return next(err);
-            }
-
-            let copyFromStream = copyFrom(sql);
-            const pgstream = client.query(copyFromStream);
-            pgstream
-                .on('error', err => {
-                    req.unpipe(pgstream);
-                    return next(err);
-                })
-                .on('end', function () {
-                    res.body = {
-                        time: (Date.now() - startTime) / 1000,
-                        total_rows: copyFromStream.rowCount
-                    };
-
-                    return next();
-                });
-
-            let requestEnded = false;
-
-            if (req.get('content-encoding') === 'gzip') {
-                req = req.pipe(zlib.createGunzip());
-            }
-
-            req
-                .on('error', err => {
-                    req.unpipe(pgstream);
-                    pgstream.end();
-                    return next(err);
-                })
-                .on('close', () => {
-                    if (!requestEnded) {
-                        req.unpipe(pgstream);
-                        pgstream.end();
-                        return next(new Error('Connection closed by client'));
+        copyCommand.pgCopyFrom(
+            req, 
+            req.query.q, 
+            res.locals.userDbParams, 
+            req.get('content-encoding') === 'gzip', 
+            function(err, result, metrics) {
+                if (err) {
+                    if (res.headersSent) {
+                        const errorHandler = errorHandlerFactory(err);
+                        res.write(JSON.stringify(errorHandler.getResponse()));
+                        res.end();
+                    } else {
+                        return next(err);
                     }
-                })
-                .on('data', data => res.locals.copyFromSize += data.length)
-                .on('end', () => requestEnded = true)
-                .pipe(pgstream);
-        });
+                } else {
+                    logger.info(metrics);
+
+                    // TODO: remove when data-ingestion log works
+                    if (req.profiler) {
+                        req.profiler.add({ copyFrom: metrics });	
+                        res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());    
+                    }
+
+                    res.body = result;
+
+                    next();
+                }
+            }
+        )
     };
 }
 
-function responseCopyFrom (logger) {
+function responseCopyFrom () {
     return function responseCopyFromMiddleware (req, res, next) {
         if (!res.body || !res.body.total_rows) {
             return next(new Error("No rows copied"));
         }
-
-        const metrics = {
-            type: 'copyfrom',
-            size: res.locals.copyFromSize, //bytes
-            format: getFormatFromCopyQuery(req.query.q),
-            time: res.body.time, //seconds
-            total_rows: res.body.total_rows, 
-            gzip: req.get('content-encoding') === 'gzip'
-        };
-        
-        logger.info(metrics);
-
-        // TODO: remove when data-ingestion log works
-        if (req.profiler) {
-            req.profiler.add({ copyFrom: metrics });	
-            res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());    
-        }
         
         res.send(res.body);
     };
@@ -207,5 +146,4 @@ function validateCopyQuery () {
     };
 }
 
-
 module.exports = CopyController;
diff --git a/app/services/copy_command.js b/app/services/copy_command.js
new file mode 100644
index 00000000..92e10eb2
--- /dev/null
+++ b/app/services/copy_command.js
@@ -0,0 +1,105 @@
+const zlib = require('zlib');
+const PSQL = require('cartodb-psql');
+const copyTo = require('pg-copy-streams').to;
+const copyFrom = require('pg-copy-streams').from;
+const { getFormatFromCopyQuery } = require('../utils/query_info');
+
+module.exports = {
+    streamCopyTo (res, sql, userDbParams, cb) {
+        let metrics = {
+            type: 'copyto',
+            size: 0, //bytes
+            time: null, //seconds
+            format: getFormatFromCopyQuery(sql),
+            total_rows: null
+        };
+
+        const startTime = Date.now();
+
+        const pg = new PSQL(userDbParams);
+        pg.connect(function (err, client) {
+            if (err) {
+                return cb(err);
+            }
+
+            const copyToStream = copyTo(sql);
+            const pgstream = client.query(copyToStream);
+            pgstream
+                .on('error', err => {
+                    pgstream.unpipe(res);
+                    return cb(err);
+                })
+                .on('data', data => metrics.size += data.length)
+                .on('end', () => {
+                    metrics.time = (Date.now() - startTime) / 1000;
+                    metrics.total_rows = copyToStream.rowCount;
+                    return cb(null, metrics);
+                })
+                .pipe(res);
+        });
+    },
+
+    streamCopyFrom (req, sql, userDbParams, isGziped, cb) {
+        let metrics = {
+            type: 'copyfrom',
+            size: 0, //bytes
+            time: null, //seconds
+            format: getFormatFromCopyQuery(sql),
+            total_rows: null, 
+            gzip: isGziped
+        }
+
+        const startTime = Date.now();
+        
+        const pg = new PSQL(userDbParams);
+        pg.connect(function (err, client) {
+            if (err) {
+                return cb(err);
+            }
+
+            let copyFromStream = copyFrom(sql);
+            const pgstream = client.query(copyFromStream);
+            pgstream
+                .on('error', err => {
+                    req.unpipe(pgstream);
+                    return cb(err);
+                })
+                .on('end', function () {
+                    metrics.time = (Date.now() - startTime) / 1000;
+                    metrics.total_rows = copyFromStream.rowCount;
+
+                    return cb(
+                        null, 
+                        {
+                            time: metrics.time,
+                            total_rows: copyFromStream.rowCount
+                        },
+                        metrics
+                    );
+                });
+
+            let requestEnded = false;
+
+            if (isGziped) {
+                req = req.pipe(zlib.createGunzip());
+            }
+
+            req
+                .on('error', err => {
+                    req.unpipe(pgstream);
+                    pgstream.end();
+                    return cb(err);
+                })
+                .on('close', () => {
+                    if (!requestEnded) {
+                        req.unpipe(pgstream);
+                        pgstream.end();
+                        return cb(new Error('Connection closed by client'));
+                    }
+                })
+                .on('data', data => metrics.size += data.length)
+                .on('end', () => requestEnded = true)
+                .pipe(pgstream);
+        });
+    }
+}

From 1fa5afd759a64974bfc7d4c82ea67fb2b794c2a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 17:50:59 +0200
Subject: [PATCH 093/133] unify error handler

---
 app/controllers/copy_controller.js | 60 ++++++++++++++++--------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index e133d49e..4b151071 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -35,6 +35,7 @@ CopyController.prototype.route = function (app) {
             validateCopyQuery(),
             handleCopyFrom(this.logger),
             responseCopyFrom(),
+            errorHandler(),
             errorMiddleware()
         ];
     };
@@ -49,6 +50,7 @@ CopyController.prototype.route = function (app) {
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
             handleCopyTo(this.logger),
+            errorHandler(),
             errorMiddleware()
         ];
     };
@@ -70,50 +72,42 @@ function handleCopyTo (logger) {
             req.query.q, 
             res.locals.userDbParams, 
             function(err, metrics) {
-            if (err) {
-                if (res.headersSent) {
-                    const errorHandler = errorHandlerFactory(err);
-                    res.write(JSON.stringify(errorHandler.getResponse()));
-                    res.end();
-                } else {
+                if (err) {
                     return next(err);
                 }
-            } else {
+                
                 logger.info(metrics);
+                // this is a especial endpoint
+                // the data from postgres is streamed to response directly
+                // next() no needed
             }
-        });
+        );
     };
 }
 
-function streamCopyFrom (logger) {
+function handleCopyFrom (logger) {
     return function handleCopyFromMiddleware (req, res, next) {
-        copyCommand.pgCopyFrom(
+        copyCommand.streamCopyFrom(
             req, 
             req.query.q, 
             res.locals.userDbParams, 
             req.get('content-encoding') === 'gzip', 
             function(err, result, metrics) {
                 if (err) {
-                    if (res.headersSent) {
-                        const errorHandler = errorHandlerFactory(err);
-                        res.write(JSON.stringify(errorHandler.getResponse()));
-                        res.end();
-                    } else {
-                        return next(err);
-                    }
-                } else {
-                    logger.info(metrics);
+                    return next(err);
+                } 
 
-                    // TODO: remove when data-ingestion log works
-                    if (req.profiler) {
-                        req.profiler.add({ copyFrom: metrics });	
-                        res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());    
-                    }
+                logger.info(metrics);
 
-                    res.body = result;
-
-                    next();
+                // TODO: remove when data-ingestion log works
+                if (req.profiler) {
+                    req.profiler.add({ copyFrom: metrics });	
+                    res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());    
                 }
+
+                res.body = result;
+
+                next();
             }
         )
     };
@@ -146,4 +140,16 @@ function validateCopyQuery () {
     };
 }
 
+function errorHandler () {
+    return function errorHandlerMiddleware (err, req, res, next) {
+        if (res.headersSent) {
+            const errorHandler = errorHandlerFactory(err);
+            res.write(JSON.stringify(errorHandler.getResponse()));
+            res.end();
+        } else {
+            return next(err);
+        }
+    };
+}
+
 module.exports = CopyController;

From b31984cbc633e8b51edfec0dc2fa9720f990a15f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 17:57:40 +0200
Subject: [PATCH 094/133] simplify response

---
 app/controllers/copy_controller.js | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 4b151071..e5287460 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -34,7 +34,6 @@ CopyController.prototype.route = function (app) {
             timeoutLimitsMiddleware(this.metadataBackend),
             validateCopyQuery(),
             handleCopyFrom(this.logger),
-            responseCopyFrom(),
             errorHandler(),
             errorMiddleware()
         ];
@@ -79,7 +78,6 @@ function handleCopyTo (logger) {
                 logger.info(metrics);
                 // this is a especial endpoint
                 // the data from postgres is streamed to response directly
-                // next() no needed
             }
         );
     };
@@ -97,6 +95,10 @@ function handleCopyFrom (logger) {
                     return next(err);
                 } 
 
+                if (!result || !result.total_rows) {
+                    return next(new Error("No rows copied"));
+                }
+                
                 logger.info(metrics);
 
                 // TODO: remove when data-ingestion log works
@@ -105,24 +107,12 @@ function handleCopyFrom (logger) {
                     res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());    
                 }
 
-                res.body = result;
-
-                next();
+                res.send(result);
             }
         )
     };
 }
 
-function responseCopyFrom () {
-    return function responseCopyFromMiddleware (req, res, next) {
-        if (!res.body || !res.body.total_rows) {
-            return next(new Error("No rows copied"));
-        }
-        
-        res.send(res.body);
-    };
-}
-
 function validateCopyQuery () {
     return function validateCopyQueryMiddleware (req, res, next) {
         const sql = req.query.q;

From 22caa049eedd1a9adcafe57f69080c5a7acc7411 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 18:47:41 +0200
Subject: [PATCH 095/133] StreamCopyMetrics

---
 app/controllers/copy_controller.js            | 27 ++++++-----
 .../{copy_command.js => stream_copy.js}       | 48 +++++--------------
 app/services/stream_copy_metrics.js           | 36 ++++++++++++++
 test/acceptance/copy-endpoints.js             |  6 +--
 4 files changed, 66 insertions(+), 51 deletions(-)
 rename app/services/{copy_command.js => stream_copy.js} (59%)
 create mode 100644 app/services/stream_copy_metrics.js

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index e5287460..498391bc 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -10,7 +10,7 @@ const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
 const BunyanLogger = require('../services/bunyanLogger');
 const errorHandlerFactory = require('../services/error_handler_factory');
-const copyCommand = require('../services/copy_command');
+const streamCopy = require('../services/stream_copy');
 
 function CopyController(metadataBackend, userDatabaseService, userLimitsService, statsClient) {
     this.metadataBackend = metadataBackend;
@@ -66,16 +66,16 @@ function handleCopyTo (logger) {
         res.header("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
         res.header("Content-Type", "application/octet-stream");
 
-        copyCommand.streamCopyTo(
+        streamCopy.to(
             res, 
             req.query.q, 
             res.locals.userDbParams, 
+            logger,
             function(err, metrics) {
                 if (err) {
                     return next(err);
                 }
                 
-                logger.info(metrics);
                 // this is a especial endpoint
                 // the data from postgres is streamed to response directly
             }
@@ -85,29 +85,34 @@ function handleCopyTo (logger) {
 
 function handleCopyFrom (logger) {
     return function handleCopyFromMiddleware (req, res, next) {
-        copyCommand.streamCopyFrom(
+        streamCopy.from(
             req, 
             req.query.q, 
             res.locals.userDbParams, 
             req.get('content-encoding') === 'gzip', 
-            function(err, result, metrics) {
+            logger,
+            function(err, response) {  // TODO: remove when data-ingestion log works: {time, rows}
                 if (err) {
                     return next(err);
                 } 
 
-                if (!result || !result.total_rows) {
+                // TODO: remove when data-ingestion log works
+                const { time, rows, type, format, gzip, size } = response; 
+
+                if (!time || !rows) {
                     return next(new Error("No rows copied"));
                 }
-                
-                logger.info(metrics);
 
                 // TODO: remove when data-ingestion log works
                 if (req.profiler) {
-                    req.profiler.add({ copyFrom: metrics });	
+                    req.profiler.add({copyFrom: { type, format, gzip, size, rows, time }});
                     res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());    
                 }
-
-                res.send(result);
+                
+                res.send({
+                    time,
+                    total_rows: rows
+                });
             }
         )
     };
diff --git a/app/services/copy_command.js b/app/services/stream_copy.js
similarity index 59%
rename from app/services/copy_command.js
rename to app/services/stream_copy.js
index 92e10eb2..ed5d0ad9 100644
--- a/app/services/copy_command.js
+++ b/app/services/stream_copy.js
@@ -2,19 +2,11 @@ const zlib = require('zlib');
 const PSQL = require('cartodb-psql');
 const copyTo = require('pg-copy-streams').to;
 const copyFrom = require('pg-copy-streams').from;
-const { getFormatFromCopyQuery } = require('../utils/query_info');
+const StreamCopyMetrics = require('./stream_copy_metrics');
 
 module.exports = {
-    streamCopyTo (res, sql, userDbParams, cb) {
-        let metrics = {
-            type: 'copyto',
-            size: 0, //bytes
-            time: null, //seconds
-            format: getFormatFromCopyQuery(sql),
-            total_rows: null
-        };
-
-        const startTime = Date.now();
+    to (res, sql, userDbParams, logger, cb) {
+        let metrics = new StreamCopyMetrics(logger, 'copyto', sql)
 
         const pg = new PSQL(userDbParams);
         pg.connect(function (err, client) {
@@ -29,27 +21,17 @@ module.exports = {
                     pgstream.unpipe(res);
                     return cb(err);
                 })
-                .on('data', data => metrics.size += data.length)
+                .on('data', data => metrics.addSize(data.length))
                 .on('end', () => {
-                    metrics.time = (Date.now() - startTime) / 1000;
-                    metrics.total_rows = copyToStream.rowCount;
+                    metrics.end(copyToStream.rowCount);
                     return cb(null, metrics);
                 })
                 .pipe(res);
         });
     },
 
-    streamCopyFrom (req, sql, userDbParams, isGziped, cb) {
-        let metrics = {
-            type: 'copyfrom',
-            size: 0, //bytes
-            time: null, //seconds
-            format: getFormatFromCopyQuery(sql),
-            total_rows: null, 
-            gzip: isGziped
-        }
-
-        const startTime = Date.now();
+    from (req, sql, userDbParams, gzip, logger, cb) {
+        let metrics = new StreamCopyMetrics(logger, 'copyfrom', sql, gzip)
         
         const pg = new PSQL(userDbParams);
         pg.connect(function (err, client) {
@@ -65,22 +47,13 @@ module.exports = {
                     return cb(err);
                 })
                 .on('end', function () {
-                    metrics.time = (Date.now() - startTime) / 1000;
-                    metrics.total_rows = copyFromStream.rowCount;
-
-                    return cb(
-                        null, 
-                        {
-                            time: metrics.time,
-                            total_rows: copyFromStream.rowCount
-                        },
-                        metrics
-                    );
+                    metrics.end(copyFromStream.rowCount);
+                    return cb(null, metrics);
                 });
 
             let requestEnded = false;
 
-            if (isGziped) {
+            if (gzip) {
                 req = req.pipe(zlib.createGunzip());
             }
 
@@ -103,3 +76,4 @@ module.exports = {
         });
     }
 }
+
diff --git a/app/services/stream_copy_metrics.js b/app/services/stream_copy_metrics.js
new file mode 100644
index 00000000..272a0b3c
--- /dev/null
+++ b/app/services/stream_copy_metrics.js
@@ -0,0 +1,36 @@
+const { getFormatFromCopyQuery } = require('../utils/query_info');
+
+module.exports = class StreamCopyMetrics {
+    constructor(logger, type, sql, gzip = null) {
+        this.logger = logger;
+
+        this.type =  type;
+        this.format = getFormatFromCopyQuery(sql);
+        this.gzip = gzip;
+        this.size = 0;
+        this.rows = 0;
+
+        this.startTime = Date.now();
+        this.endTime;
+        this.time;
+    }
+
+    addSize (size) {
+        this.size += size;
+    }
+
+    end (rows = null) {
+        this.rows = rows;
+        this.endTime = Date.now();
+        this.time = (this.endTime - this.startTime) / 1000;
+
+        this.logger.info({
+            type: this.type,
+            format: this.format,
+            gzip: this.gzip,
+            size: this.size,
+            rows: this.rows,
+            time: this.time
+        });
+    }
+}
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 98566ffe..2363a98e 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -17,7 +17,7 @@ const statsClient = StatsClient.getInstance(global.settings.statsd);
 const server = require('../../app/server')(statsClient);
 
 
-describe('copy-endpoints', function() {
+describe.only('copy-endpoints', function() {
     it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({
@@ -39,7 +39,7 @@ describe('copy-endpoints', function() {
             assert.equal(metrics.size, 57);
             assert.equal(metrics.format, 'CSV');
             assert.equal(metrics.time, response.time);
-            assert.equal(metrics.total_rows, response.total_rows);
+            assert.equal(metrics.rows, response.total_rows);
             assert.equal(metrics.gzip, false);
 
             done();
@@ -168,7 +168,7 @@ describe('copy-endpoints', function() {
             assert.equal(metrics.size, 57);
             assert.equal(metrics.format, 'CSV');
             assert.equal(metrics.time, response.time);
-            assert.equal(metrics.total_rows, response.total_rows);
+            assert.equal(metrics.rows, response.total_rows);
             assert.equal(metrics.gzip, true);
             
             done();

From 3cf28bb7ffb422a213c67d5cc5e25cf0bc5eb53a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 25 May 2018 18:50:56 +0200
Subject: [PATCH 096/133] jshint happy

---
 app/controllers/copy_controller.js  | 8 ++++----
 app/services/stream_copy.js         | 7 +++----
 app/services/stream_copy_metrics.js | 6 +++---
 test/acceptance/copy-endpoints.js   | 2 +-
 4 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 498391bc..c8f8c863 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -71,7 +71,7 @@ function handleCopyTo (logger) {
             req.query.q, 
             res.locals.userDbParams, 
             logger,
-            function(err, metrics) {
+            function(err) {
                 if (err) {
                     return next(err);
                 }
@@ -91,13 +91,13 @@ function handleCopyFrom (logger) {
             res.locals.userDbParams, 
             req.get('content-encoding') === 'gzip', 
             logger,
-            function(err, response) {  // TODO: remove when data-ingestion log works: {time, rows}
+            function(err, metrics) {  // TODO: remove when data-ingestion log works: {time, rows}
                 if (err) {
                     return next(err);
                 } 
 
                 // TODO: remove when data-ingestion log works
-                const { time, rows, type, format, gzip, size } = response; 
+                const { time, rows, type, format, gzip, size } = metrics; 
 
                 if (!time || !rows) {
                     return next(new Error("No rows copied"));
@@ -114,7 +114,7 @@ function handleCopyFrom (logger) {
                     total_rows: rows
                 });
             }
-        )
+        );
     };
 }
 
diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index ed5d0ad9..c0a4f728 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -6,7 +6,7 @@ const StreamCopyMetrics = require('./stream_copy_metrics');
 
 module.exports = {
     to (res, sql, userDbParams, logger, cb) {
-        let metrics = new StreamCopyMetrics(logger, 'copyto', sql)
+        let metrics = new StreamCopyMetrics(logger, 'copyto', sql);
 
         const pg = new PSQL(userDbParams);
         pg.connect(function (err, client) {
@@ -31,7 +31,7 @@ module.exports = {
     },
 
     from (req, sql, userDbParams, gzip, logger, cb) {
-        let metrics = new StreamCopyMetrics(logger, 'copyfrom', sql, gzip)
+        let metrics = new StreamCopyMetrics(logger, 'copyfrom', sql, gzip);
         
         const pg = new PSQL(userDbParams);
         pg.connect(function (err, client) {
@@ -75,5 +75,4 @@ module.exports = {
                 .pipe(pgstream);
         });
     }
-}
-
+};
diff --git a/app/services/stream_copy_metrics.js b/app/services/stream_copy_metrics.js
index 272a0b3c..d61f9bec 100644
--- a/app/services/stream_copy_metrics.js
+++ b/app/services/stream_copy_metrics.js
@@ -11,8 +11,8 @@ module.exports = class StreamCopyMetrics {
         this.rows = 0;
 
         this.startTime = Date.now();
-        this.endTime;
-        this.time;
+        this.endTime = null;
+        this.time = null;
     }
 
     addSize (size) {
@@ -33,4 +33,4 @@ module.exports = class StreamCopyMetrics {
             time: this.time
         });
     }
-}
+};
diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 2363a98e..fe52bd58 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -17,7 +17,7 @@ const statsClient = StatsClient.getInstance(global.settings.statsd);
 const server = require('../../app/server')(statsClient);
 
 
-describe.only('copy-endpoints', function() {
+describe('copy-endpoints', function() {
     it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({

From 7d32ae293f39200d2c5afe8e67bc7d27ca6209aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 28 May 2018 11:18:30 +0200
Subject: [PATCH 097/133] rename buyan_logger file name

---
 app/controllers/copy_controller.js                 | 2 +-
 app/services/{bunyanLogger.js => bunyan_logger.js} | 0
 batch/batch-logger.js                              | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename app/services/{bunyanLogger.js => bunyan_logger.js} (100%)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index c8f8c863..cd58194d 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -8,7 +8,7 @@ const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
 const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
-const BunyanLogger = require('../services/bunyanLogger');
+const BunyanLogger = require('../services/bunyan_logger');
 const errorHandlerFactory = require('../services/error_handler_factory');
 const streamCopy = require('../services/stream_copy');
 
diff --git a/app/services/bunyanLogger.js b/app/services/bunyan_logger.js
similarity index 100%
rename from app/services/bunyanLogger.js
rename to app/services/bunyan_logger.js
diff --git a/batch/batch-logger.js b/batch/batch-logger.js
index b424a7cc..67b7aafc 100644
--- a/batch/batch-logger.js
+++ b/batch/batch-logger.js
@@ -1,6 +1,6 @@
 'use strict';
 
-const BunyanLogger = require('../app/services/bunyanLogger');
+const BunyanLogger = require('../app/services/bunyan_logger');
 
 class BatchLogger extends BunyanLogger {
     constructor (path, name) {

From e0926472464630141e562be35d7aac4bc0666181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 28 May 2018 11:25:09 +0200
Subject: [PATCH 098/133] ensuring data ingestion log

---
 app/controllers/copy_controller.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index cd58194d..d4ca0a7c 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -18,7 +18,10 @@ function CopyController(metadataBackend, userDatabaseService, userLimitsService,
     this.userLimitsService = userLimitsService;
     this.statsClient = statsClient;
 
-    this.logger = new BunyanLogger(global.settings.dataIngestionLogPath, 'data-ingestion');
+    this.logger = new BunyanLogger(
+        global.settings.dataIngestionLogPath || 'log/data-ingestion.log', 
+        'data-ingestion'
+    );
 }
 
 CopyController.prototype.route = function (app) {

From 008fad3d137ef9181b435d98bc155cb715efbb4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 29 May 2018 12:45:57 +0200
Subject: [PATCH 099/133] undo 'ensuring data ingestion log'

---
 app/controllers/copy_controller.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index d4ca0a7c..cd58194d 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -18,10 +18,7 @@ function CopyController(metadataBackend, userDatabaseService, userLimitsService,
     this.userLimitsService = userLimitsService;
     this.statsClient = statsClient;
 
-    this.logger = new BunyanLogger(
-        global.settings.dataIngestionLogPath || 'log/data-ingestion.log', 
-        'data-ingestion'
-    );
+    this.logger = new BunyanLogger(global.settings.dataIngestionLogPath, 'data-ingestion');
 }
 
 CopyController.prototype.route = function (app) {

From 5f8533bf99c91583acb9eb1c4ae31c5707274479 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 29 May 2018 16:19:06 +0200
Subject: [PATCH 100/133] get the size gzipped

---
 test/acceptance/copy-endpoints.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index fe52bd58..f750986d 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -165,7 +165,7 @@ describe('copy-endpoints', function() {
             const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
             assert.ok(headers.copyFrom);
             const metrics = headers.copyFrom;
-            assert.equal(metrics.size, 57);
+            assert.equal(metrics.size, 96);
             assert.equal(metrics.format, 'CSV');
             assert.equal(metrics.time, response.time);
             assert.equal(metrics.rows, response.total_rows);

From a083eb909c710c0e3323d421cc0d4687578ff47a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 29 May 2018 16:19:53 +0200
Subject: [PATCH 101/133] fix req events

---
 app/services/stream_copy.js | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index c0a4f728..5207e967 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -44,7 +44,7 @@ module.exports = {
             pgstream
                 .on('error', err => {
                     req.unpipe(pgstream);
-                    return cb(err);
+                    return cb(err); 
                 })
                 .on('end', function () {
                     metrics.end(copyFromStream.rowCount);
@@ -53,10 +53,6 @@ module.exports = {
 
             let requestEnded = false;
 
-            if (gzip) {
-                req = req.pipe(zlib.createGunzip());
-            }
-
             req
                 .on('error', err => {
                     req.unpipe(pgstream);
@@ -71,8 +67,13 @@ module.exports = {
                     }
                 })
                 .on('data', data => metrics.size += data.length)
-                .on('end', () => requestEnded = true)
-                .pipe(pgstream);
+                .on('end', () => requestEnded = true);
+
+            if (gzip) {
+                req.pipe(zlib.createGunzip()).pipe(pgstream);
+            } else {
+                req.pipe(pgstream);
+            }
         });
     }
 };

From 310f652ae418454d52bb88a16235aaf00bf95399 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Wed, 30 May 2018 12:59:49 +0200
Subject: [PATCH 102/133] send CopyFail when user close connection

---
 app/services/stream_copy.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 5207e967..b7387a0e 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -61,8 +61,9 @@ module.exports = {
                 })
                 .on('close', () => {
                     if (!requestEnded) {
+                        const connection = client.connection;
+                        connection.sendCopyFail();
                         req.unpipe(pgstream);
-                        pgstream.end();
                         return cb(new Error('Connection closed by client'));
                     }
                 })

From 0eab03a7e76ff4e02f8d77d5e0647e2dc8e61748 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Thu, 31 May 2018 16:41:22 +0200
Subject: [PATCH 103/133] Add a more informative message

---
 app/services/stream_copy.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index b7387a0e..273a1a33 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -62,7 +62,7 @@ module.exports = {
                 .on('close', () => {
                     if (!requestEnded) {
                         const connection = client.connection;
-                        connection.sendCopyFail();
+                        connection.sendCopyFail('CARTO SQL API: Connection closed by client');
                         req.unpipe(pgstream);
                         return cb(new Error('Connection closed by client'));
                     }

From 332f7096d380dea5e54ef7d6287cc74546a1f1ca Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Thu, 31 May 2018 17:06:17 +0200
Subject: [PATCH 104/133] Listen to response events (on behalf of @oleurud)

---
 app/services/stream_copy.js | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 273a1a33..0427bfb3 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -14,6 +14,21 @@ module.exports = {
                 return cb(err);
             }
 
+            let responseEnded = false;
+
+            res
+                .on('error', err => {
+                    pgstream.unpipe(res);
+                    return cb(err);
+                })
+                .on('close', () => {
+                    if (!responseEnded) {
+
+                        return cb(new Error('Connection closed by client'));
+                    }
+                })
+                .on('end', () => responseEnded = true);
+
             const copyToStream = copyTo(sql);
             const pgstream = client.query(copyToStream);
             pgstream

From b59ae1d057fb349a4b921fc365eb6ce2f5fac900 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Thu, 31 May 2018 17:27:35 +0200
Subject: [PATCH 105/133] Make sure the COPY TO query is cancelled

Issue a CancelRequest upon client disconnection, to make sure the COPY
TO query is cancelled and the connection/session is put back to the pg
pool.
---
 app/services/stream_copy.js | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 0427bfb3..b5548bae 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -3,6 +3,7 @@ const PSQL = require('cartodb-psql');
 const copyTo = require('pg-copy-streams').to;
 const copyFrom = require('pg-copy-streams').from;
 const StreamCopyMetrics = require('./stream_copy_metrics');
+const { Client } = require('pg');
 
 module.exports = {
     to (res, sql, userDbParams, logger, cb) {
@@ -23,6 +24,15 @@ module.exports = {
                 })
                 .on('close', () => {
                     if (!responseEnded) {
+                        // Cancel the running COPY TO query.
+                        // See https://www.postgresql.org/docs/9.5/static/protocol-flow.html#PROTOCOL-COPY
+                        const runningClient = client;
+                        const cancelingClient = new Client(runningClient.connectionParameters);
+                        const connection = cancelingClient.connection;
+                        connection.connect(runningClient.port, runningClient.host);
+                        connection.on('connect', () => {
+                            connection.cancel(runningClient.processID, runningClient.secretKey);
+                        });
 
                         return cb(new Error('Connection closed by client'));
                     }

From 994e8a702bd6fda13e42453821bdf621fe4a4e1e Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Thu, 31 May 2018 18:59:17 +0200
Subject: [PATCH 106/133] Add callbacks to pg.connect

And call them to return connections to the pool.
---
 app/services/stream_copy.js | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index b5548bae..3c3cc143 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -10,7 +10,7 @@ module.exports = {
         let metrics = new StreamCopyMetrics(logger, 'copyto', sql);
 
         const pg = new PSQL(userDbParams);
-        pg.connect(function (err, client) {
+        pg.connect(function (err, client, done) {
             if (err) {
                 return cb(err);
             }
@@ -20,6 +20,7 @@ module.exports = {
             res
                 .on('error', err => {
                     pgstream.unpipe(res);
+                    done();
                     return cb(err);
                 })
                 .on('close', () => {
@@ -29,11 +30,11 @@ module.exports = {
                         const runningClient = client;
                         const cancelingClient = new Client(runningClient.connectionParameters);
                         const connection = cancelingClient.connection;
-                        connection.connect(runningClient.port, runningClient.host);
                         connection.on('connect', () => {
                             connection.cancel(runningClient.processID, runningClient.secretKey);
+                            done();
                         });
-
+                        connection.connect(runningClient.port, runningClient.host);
                         return cb(new Error('Connection closed by client'));
                     }
                 })
@@ -44,11 +45,13 @@ module.exports = {
             pgstream
                 .on('error', err => {
                     pgstream.unpipe(res);
+                    done();
                     return cb(err);
                 })
                 .on('data', data => metrics.addSize(data.length))
                 .on('end', () => {
                     metrics.end(copyToStream.rowCount);
+                    done();
                     return cb(null, metrics);
                 })
                 .pipe(res);
@@ -59,7 +62,7 @@ module.exports = {
         let metrics = new StreamCopyMetrics(logger, 'copyfrom', sql, gzip);
         
         const pg = new PSQL(userDbParams);
-        pg.connect(function (err, client) {
+        pg.connect(function (err, client, done) {
             if (err) {
                 return cb(err);
             }
@@ -69,10 +72,12 @@ module.exports = {
             pgstream
                 .on('error', err => {
                     req.unpipe(pgstream);
+                    done();
                     return cb(err); 
                 })
                 .on('end', function () {
                     metrics.end(copyFromStream.rowCount);
+                    done();
                     return cb(null, metrics);
                 });
 
@@ -82,6 +87,7 @@ module.exports = {
                 .on('error', err => {
                     req.unpipe(pgstream);
                     pgstream.end();
+                    done();
                     return cb(err);
                 })
                 .on('close', () => {
@@ -89,6 +95,7 @@ module.exports = {
                         const connection = client.connection;
                         connection.sendCopyFail('CARTO SQL API: Connection closed by client');
                         req.unpipe(pgstream);
+                        done();
                         return cb(new Error('Connection closed by client'));
                     }
                 })

From b05ded92aa02ce80d983314d6ef6038e514ec493 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 1 Jun 2018 11:26:28 +0200
Subject: [PATCH 107/133] db connections usage test

---
 test/acceptance/copy-endpoints.js | 62 ++++++++++++++++++++++++++++++-
 1 file changed, 61 insertions(+), 1 deletion(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index f750986d..45b11ba0 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -178,7 +178,7 @@ describe('copy-endpoints', function() {
 });
 
 
-describe('copy-endpoints timeout', function() {
+describe('copy-endpoints timeout', function() {       
     it('should fail with copyfrom and timeout', function(done){
         assert.response(server, {
             url: '/api/v1/sql?q=set statement_timeout = 10',
@@ -255,3 +255,63 @@ describe('copy-endpoints timeout', function() {
         });
     });
 });
+
+
+describe('copy-endpoints db connections', function() {
+    before(function() {
+        this.db_pool_size = global.settings.db_pool_size;
+        global.settings.db_pool_size = 1;
+    });
+
+    after(function() {
+        global.settings.db_pool_size = this.db_pool_size;
+    });
+
+    it('copyfrom', function(done) {
+        const query = "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)";
+        function doCopyFrom() {
+            return new Promise(resolve => {
+                assert.response(server, {
+                    url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                        q: query
+                    }),
+                    data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                    headers: {host: 'vizzuality.cartodb.com'},
+                    method: 'POST'
+                },{}, function(err, res) {
+                    assert.ifError(err);
+                    const response = JSON.parse(res.body);
+                    assert.ok(response.time);
+                    resolve();
+                });
+            });
+        }
+
+        Promise.all([doCopyFrom(), doCopyFrom(), doCopyFrom()]).then(function() {
+            done();
+        });
+    });
+
+    it('copyto', function(done) {
+        function doCopyTo() {
+            return new Promise(resolve => {
+                assert.response(server, {
+                    url: "/api/v1/sql/copyto?" + querystring.stringify({
+                        q: 'COPY copy_endpoints_test TO STDOUT',
+                        filename: '/tmp/output.dmp'
+                    }),
+                    headers: {host: 'vizzuality.cartodb.com'},
+                    method: 'GET'
+                },{}, function(err, res) {
+                    assert.ifError(err);
+                    assert.ok(res.body);
+                    resolve();
+                });
+            });
+        }
+
+        Promise.all([doCopyTo(), doCopyTo(), doCopyTo()]).then(function() {
+            done();
+        });
+    });
+});

From 4022fb2967ea61b67ad426da76b52d0f77a0d4fd Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 4 Jun 2018 15:46:39 +0200
Subject: [PATCH 108/133] Clean up before executing the copy suite

So that it can be executed saving a bit of setup/teardown time:

  test/run_tests.sh --nodrop --nocreate test/acceptance/copy-endpoints.js
---
 test/acceptance/copy-endpoints.js | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 45b11ba0..4a8ef9cc 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -4,6 +4,7 @@ const fs = require('fs');
 const querystring = require('querystring');
 const assert = require('../support/assert');
 const os = require('os');
+const { Client } = require('pg');
 
 const StatsClient = require('../../app/stats/client');
 if (global.settings.statsd) {
@@ -18,6 +19,20 @@ const server = require('../../app/server')(statsClient);
 
 
 describe('copy-endpoints', function() {
+    before(function(done) {
+        const client = new Client({
+            user: 'postgres',
+            host: 'localhost',
+            database: 'cartodb_test_user_1_db',
+            port: 5432,
+        });
+        client.connect();
+        client.query('TRUNCATE copy_endpoints_test', (err/*, res */) => {
+            client.end();
+            done(err);
+        });
+    });
+
     it('should work with copyfrom endpoint', function(done){
         assert.response(server, {
             url: "/api/v1/sql/copyfrom?" + querystring.stringify({

From 1cf7032c9a0815c566b6eaafaaf01bb08027589f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 4 Jun 2018 18:08:34 +0200
Subject: [PATCH 109/133] adding user and date in copy logs

---
 app/controllers/copy_controller.js  |  4 ++-
 app/services/stream_copy.js         |  8 ++---
 app/services/stream_copy_metrics.js | 49 ++++++++++++++++++++++-------
 3 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index cd58194d..0bc652c1 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -69,7 +69,8 @@ function handleCopyTo (logger) {
         streamCopy.to(
             res, 
             req.query.q, 
-            res.locals.userDbParams, 
+            res.locals.userDbParams,
+            res.locals.user,
             logger,
             function(err) {
                 if (err) {
@@ -89,6 +90,7 @@ function handleCopyFrom (logger) {
             req, 
             req.query.q, 
             res.locals.userDbParams, 
+            res.locals.user,
             req.get('content-encoding') === 'gzip', 
             logger,
             function(err, metrics) {  // TODO: remove when data-ingestion log works: {time, rows}
diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index b7387a0e..4b6882df 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -5,8 +5,8 @@ const copyFrom = require('pg-copy-streams').from;
 const StreamCopyMetrics = require('./stream_copy_metrics');
 
 module.exports = {
-    to (res, sql, userDbParams, logger, cb) {
-        let metrics = new StreamCopyMetrics(logger, 'copyto', sql);
+    to (res, sql, userDbParams, user, logger, cb) {
+        let metrics = new StreamCopyMetrics(logger, 'copyto', sql, user);
 
         const pg = new PSQL(userDbParams);
         pg.connect(function (err, client) {
@@ -30,8 +30,8 @@ module.exports = {
         });
     },
 
-    from (req, sql, userDbParams, gzip, logger, cb) {
-        let metrics = new StreamCopyMetrics(logger, 'copyfrom', sql, gzip);
+    from (req, sql, userDbParams, user, gzip, logger, cb) {
+        let metrics = new StreamCopyMetrics(logger, 'copyfrom', sql, user, gzip);
         
         const pg = new PSQL(userDbParams);
         pg.connect(function (err, client) {
diff --git a/app/services/stream_copy_metrics.js b/app/services/stream_copy_metrics.js
index d61f9bec..31ba1a37 100644
--- a/app/services/stream_copy_metrics.js
+++ b/app/services/stream_copy_metrics.js
@@ -1,36 +1,61 @@
 const { getFormatFromCopyQuery } = require('../utils/query_info');
 
 module.exports = class StreamCopyMetrics {
-    constructor(logger, type, sql, gzip = null) {
+    constructor(logger, type, sql, user, gzip = null) {
         this.logger = logger;
 
-        this.type =  type;
+        this.type = type;
         this.format = getFormatFromCopyQuery(sql);
         this.gzip = gzip;
+        this.username = user;
         this.size = 0;
         this.rows = 0;
 
-        this.startTime = Date.now();
+        this.startTime = new Date();
         this.endTime = null;
         this.time = null;
+
+        this.error = null;
     }
 
-    addSize (size) {
+    addSize(size) {
         this.size += size;
     }
 
-    end (rows = null) {
-        this.rows = rows;
-        this.endTime = Date.now();
-        this.time = (this.endTime - this.startTime) / 1000;
+    end(rows = null, error = null) {
+        if (Number.isInteger(rows)) {
+            this.rows = rows;
+        }
 
-        this.logger.info({
+        if (error instanceof Error) {
+            this.error = error;
+        }
+
+        this.endTime = new Date();
+        this.time = (this.endTime.getTime() - this.startTime.getTime()) / 1000;
+
+        this._log({
+            timestamp: this.startTime.toISOString(),
+            errorMessage: this.error ? this.error.message : null
+        })
+    }
+
+    _log({timestamp, errorMessage = null}) {
+        let logData = {
             type: this.type,
             format: this.format,
-            gzip: this.gzip,
             size: this.size,
             rows: this.rows,
-            time: this.time
-        });
+            gzip: this.gzip,
+            username: this.username,
+            time: this.time,
+            timestamp
+        };
+
+        if (errorMessage) {
+            logData.error = errorMessage;
+        }
+
+        this.logger.info(logData);
     }
 };

From a844b5d31d59bebb5acb40ad331e19cd03132f80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 4 Jun 2018 18:15:00 +0200
Subject: [PATCH 110/133] jshint happy

---
 app/services/stream_copy_metrics.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/services/stream_copy_metrics.js b/app/services/stream_copy_metrics.js
index 31ba1a37..5ae6a85b 100644
--- a/app/services/stream_copy_metrics.js
+++ b/app/services/stream_copy_metrics.js
@@ -34,13 +34,13 @@ module.exports = class StreamCopyMetrics {
         this.endTime = new Date();
         this.time = (this.endTime.getTime() - this.startTime.getTime()) / 1000;
 
-        this._log({
-            timestamp: this.startTime.toISOString(),
-            errorMessage: this.error ? this.error.message : null
-        })
+        this._log(
+            this.startTime.toISOString(),
+            this.error ? this.error.message : null
+        );
     }
 
-    _log({timestamp, errorMessage = null}) {
+    _log(timestamp, errorMessage = null) {
         let logData = {
             type: this.type,
             format: this.format,

From 2f2dcfd762eeef7198eb31a077a1cb623e4f2715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 4 Jun 2018 18:15:28 +0200
Subject: [PATCH 111/133] fix copy format case

---
 app/utils/query_info.js      | 2 +-
 test/unit/query_info.test.js | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/utils/query_info.js b/app/utils/query_info.js
index cc184dff..b56114ba 100644
--- a/app/utils/query_info.js
+++ b/app/utils/query_info.js
@@ -10,7 +10,7 @@ module.exports = {
             return false;
         }
         
-        if(copyQuery.includes(' WITH ') && copyQuery.includes('FORMAT ')) {
+        if(copyQuery.includes(' WITH') && copyQuery.includes('FORMAT ')) {
             const regex = /\bFORMAT\s+(\w+)/;
             const result = regex.exec(copyQuery);
     
diff --git a/test/unit/query_info.test.js b/test/unit/query_info.test.js
index 5f889025..d3216b99 100644
--- a/test/unit/query_info.test.js
+++ b/test/unit/query_info.test.js
@@ -9,6 +9,7 @@ describe('query info', function () {
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT  CSV, DELIMITER ',', HEADER true)",
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV , DELIMITER ',', HEADER true)",
                 "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV)",
+                "COPY copy_endpoints_test FROM STDIN WITH(FORMAT csv,HEADER true)"
             ];
 
             validQueries.forEach(query => {

From 014ea8142b572042adb0772022afdb62b676a2a7 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 4 Jun 2018 16:34:37 +0200
Subject: [PATCH 112/133] A cleaner approach to the cancel command

---
 app/services/stream_copy.js | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 3c3cc143..1b627ec9 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -16,6 +16,9 @@ module.exports = {
             }
 
             let responseEnded = false;
+            let connectionClosedByClient = false;
+            const copyToStream = copyTo(sql);
+            const pgstream = client.query(copyToStream);
 
             res
                 .on('error', err => {
@@ -25,28 +28,26 @@ module.exports = {
                 })
                 .on('close', () => {
                     if (!responseEnded) {
-                        // Cancel the running COPY TO query.
+                        connectionClosedByClient = true;
+                        // Cancel the running COPY TO query
                         // See https://www.postgresql.org/docs/9.5/static/protocol-flow.html#PROTOCOL-COPY
                         const runningClient = client;
                         const cancelingClient = new Client(runningClient.connectionParameters);
-                        const connection = cancelingClient.connection;
-                        connection.on('connect', () => {
-                            connection.cancel(runningClient.processID, runningClient.secretKey);
-                            done();
-                        });
-                        connection.connect(runningClient.port, runningClient.host);
-                        return cb(new Error('Connection closed by client'));
+                        cancelingClient.cancel(runningClient, pgstream);
+                        pgstream.unpipe(res);
+                        done(new Error('Connection closed by client'));
+                        return cb(err);
                     }
                 })
                 .on('end', () => responseEnded = true);
 
-            const copyToStream = copyTo(sql);
-            const pgstream = client.query(copyToStream);
             pgstream
                 .on('error', err => {
-                    pgstream.unpipe(res);
-                    done();
-                    return cb(err);
+                    if (!connectionClosedByClient) {
+                        pgstream.unpipe(res);
+                        done(new Error('Connection closed by client'));
+                        return cb(err);
+                    }
                 })
                 .on('data', data => metrics.addSize(data.length))
                 .on('end', () => {

From 8d486ef9673245f36cb34ff800d14ea51c4f64de Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 4 Jun 2018 19:07:03 +0200
Subject: [PATCH 113/133] Ignore log and yarn.lock

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index ed26bf95..147f2348 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,5 @@ test/test.log
 test/acceptance/oauth/venv/*
 coverage/
 npm-debug.log
+log/*.log
+yarn.lock
\ No newline at end of file

From 42cd74e5533d4313a3e569ec7889a72318af30de Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Mon, 4 Jun 2018 11:34:38 -0700
Subject: [PATCH 114/133] Change parameter lists to table, per @andye request

---
 doc/copy_queries.md | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/doc/copy_queries.md b/doc/copy_queries.md
index 7b4230c5..b08eec8c 100644
--- a/doc/copy_queries.md
+++ b/doc/copy_queries.md
@@ -18,9 +18,12 @@ If the `COPY` command, the supplied file, and the target table do not all match,
 
 "Copy from" copies data "from" your file, "to" CARTO. "Copy from" uses chunked encoding (`Transfer-Encoding: chunked`) to stream an upload file to the server. This avoids limitations around file size and any need for temporary storage: the data travels from your file straight into the database.
 
-* `api_key` provided in the request URL parameters.
-* `q` the copy SQL provided either in the request URL parameters.
-* the actual copy file content, as the body of the POST.
+Parameter | Description
+--- | ---
+`api_key` | a write-enabled key
+`q` | the `COPY` command to load the data
+
+**The actual COPY file content must be sent as the body of the POST request.**
 
 Composing a chunked POST is moderately complicated, so most developers will use a tool or scripting language to upload data to CARTO via "copy from". 
 
@@ -129,11 +132,13 @@ Using the `copyto` end point to extract data bypasses the usual JSON formatting
 * [CSV](https://www.postgresql.org/docs/10/static/sql-copy.html#id-1.9.3.52.9.3), and
 * PgSQL [binary format](https://www.postgresql.org/docs/10/static/sql-copy.html#id-1.9.3.52.9.4).
 
-"Copy to" is a simple HTTP GET end point, so any tool or language can be easily used to download data, supplying the following parameters in the URL:
+"Copy to" is a simple HTTP GET end point, so any tool or language can be easily used to download data, supplying the following parameters in the URL.
 
-* `q`, the "COPY" command to extract the data.
-* `filename`, the filename to put in the "Content-disposition" HTTP header. Useful for tools that automatically save the download to a file name.
-* `api_key`, your API key for reading non-public tables.
+Parameter | Description
+--- | ---
+`api_key` | your API key for reading non-public tables
+`q` | the `COPY` command to extract the data
+`filename` | filename to put in the "Content-disposition" HTTP header, useful for tools that automatically save the download to a file name
 
 
 ### CURL Example

From 7b6056b79947e528b6d3ffed26eaf1017e2acd74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 4 Jun 2018 20:36:16 +0200
Subject: [PATCH 115/133] using the correct errors in done

---
 app/services/stream_copy.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 1b627ec9..2c8050b9 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -34,8 +34,11 @@ module.exports = {
                         const runningClient = client;
                         const cancelingClient = new Client(runningClient.connectionParameters);
                         cancelingClient.cancel(runningClient, pgstream);
+
+                        const err = new Error('Connection closed by client');
                         pgstream.unpipe(res);
-                        done(new Error('Connection closed by client'));
+                        // see https://node-postgres.com/api/pool#release-err-error-
+                        done(err);
                         return cb(err);
                     }
                 })
@@ -45,7 +48,7 @@ module.exports = {
                 .on('error', err => {
                     if (!connectionClosedByClient) {
                         pgstream.unpipe(res);
-                        done(new Error('Connection closed by client'));
+                        done();
                         return cb(err);
                     }
                 })

From 66f7ab45fea36427948c90349b86107cd558a2ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 4 Jun 2018 20:51:21 +0200
Subject: [PATCH 116/133] release connection with error

---
 app/services/stream_copy.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 2c8050b9..15d388b7 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -37,7 +37,7 @@ module.exports = {
 
                         const err = new Error('Connection closed by client');
                         pgstream.unpipe(res);
-                        // see https://node-postgres.com/api/pool#release-err-error-
+                        // see https://node-postgres.com/api/pool#releasecallback
                         done(err);
                         return cb(err);
                     }
@@ -48,7 +48,7 @@ module.exports = {
                 .on('error', err => {
                     if (!connectionClosedByClient) {
                         pgstream.unpipe(res);
-                        done();
+                        done(err);
                         return cb(err);
                     }
                 })

From d6d9022c7fce8275c7016ed3ca45309dedb20bc9 Mon Sep 17 00:00:00 2001
From: Paul Ramsey <pramsey@cleverelephant.ca>
Date: Mon, 4 Jun 2018 13:12:06 -0700
Subject: [PATCH 117/133] Coerce format string to upper for better log
 consistency

---
 app/utils/query_info.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/utils/query_info.js b/app/utils/query_info.js
index b56114ba..c605c785 100644
--- a/app/utils/query_info.js
+++ b/app/utils/query_info.js
@@ -17,6 +17,7 @@ module.exports = {
             if (result && result.length === 2) {
                 if (COPY_FORMATS.includes(result[1])) {
                     format = result[1];
+                    format = format.toUpperCase();
                 } else {
                     format = false;
                 }

From 550c736032199ca0fa2e3c73ae010495d5ae9ce8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 5 Jun 2018 12:35:52 +0200
Subject: [PATCH 118/133] metrics in errors

---
 app/services/stream_copy.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index d85932b2..3d92ddfd 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -22,6 +22,7 @@ module.exports = {
 
             res
                 .on('error', err => {
+                    metrics.end(null, err);
                     pgstream.unpipe(res);
                     done();
                     return cb(err);
@@ -36,6 +37,7 @@ module.exports = {
                         cancelingClient.cancel(runningClient, pgstream);
 
                         const err = new Error('Connection closed by client');
+                        metrics.end(null, err);
                         pgstream.unpipe(res);
                         // see https://node-postgres.com/api/pool#releasecallback
                         done(err);
@@ -47,6 +49,7 @@ module.exports = {
             pgstream
                 .on('error', err => {
                     if (!connectionClosedByClient) {
+                        metrics.end(null, err);
                         pgstream.unpipe(res);
                         done(err);
                         return cb(err);
@@ -75,6 +78,7 @@ module.exports = {
             const pgstream = client.query(copyFromStream);
             pgstream
                 .on('error', err => {
+                    metrics.end(null, err);
                     req.unpipe(pgstream);
                     done();
                     return cb(err); 
@@ -89,6 +93,7 @@ module.exports = {
 
             req
                 .on('error', err => {
+                    metrics.end(null, err);
                     req.unpipe(pgstream);
                     pgstream.end();
                     done();
@@ -96,11 +101,13 @@ module.exports = {
                 })
                 .on('close', () => {
                     if (!requestEnded) {
+                        const err = new Error('Connection closed by client');
+                        metrics.end(null, err);
                         const connection = client.connection;
                         connection.sendCopyFail('CARTO SQL API: Connection closed by client');
                         req.unpipe(pgstream);
                         done();
-                        return cb(new Error('Connection closed by client'));
+                        return cb(err);
                     }
                 })
                 .on('data', data => metrics.size += data.length)

From dd689d3568943b48fb9c55e813966c739eb43757 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 5 Jun 2018 12:36:36 +0200
Subject: [PATCH 119/133] log metrics only once

---
 app/services/stream_copy_metrics.js | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/app/services/stream_copy_metrics.js b/app/services/stream_copy_metrics.js
index 5ae6a85b..9321b36c 100644
--- a/app/services/stream_copy_metrics.js
+++ b/app/services/stream_copy_metrics.js
@@ -16,6 +16,8 @@ module.exports = class StreamCopyMetrics {
         this.time = null;
 
         this.error = null;
+
+        this.ended = false;
     }
 
     addSize(size) {
@@ -23,6 +25,12 @@ module.exports = class StreamCopyMetrics {
     }
 
     end(rows = null, error = null) {
+        if (this.ended) {
+            return;
+        }
+        
+        this.ended = true;
+
         if (Number.isInteger(rows)) {
             this.rows = rows;
         }

From 02cc606be17d950ccf7279d7973afbc6a5eded63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 5 Jun 2018 13:01:15 +0200
Subject: [PATCH 120/133] gzipSize support in metrics

---
 app/services/stream_copy_metrics.js | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/app/services/stream_copy_metrics.js b/app/services/stream_copy_metrics.js
index 9321b36c..24143a0d 100644
--- a/app/services/stream_copy_metrics.js
+++ b/app/services/stream_copy_metrics.js
@@ -9,6 +9,7 @@ module.exports = class StreamCopyMetrics {
         this.gzip = gzip;
         this.username = user;
         this.size = 0;
+        this.gzipSize = 0;
         this.rows = 0;
 
         this.startTime = new Date();
@@ -24,11 +25,15 @@ module.exports = class StreamCopyMetrics {
         this.size += size;
     }
 
+    addGzipSize(size) {
+        this.gzipSize += size;
+    }
+
     end(rows = null, error = null) {
         if (this.ended) {
             return;
         }
-        
+
         this.ended = true;
 
         if (Number.isInteger(rows)) {
@@ -44,11 +49,12 @@ module.exports = class StreamCopyMetrics {
 
         this._log(
             this.startTime.toISOString(),
+            this.gzip && this.gzipSize ? this.gzipSize : null,
             this.error ? this.error.message : null
         );
     }
 
-    _log(timestamp, errorMessage = null) {
+    _log(timestamp, gzipSize = null, errorMessage = null) {
         let logData = {
             type: this.type,
             format: this.format,
@@ -60,6 +66,10 @@ module.exports = class StreamCopyMetrics {
             timestamp
         };
 
+        if (gzipSize) {
+            logData.gzipSize = gzipSize;
+        }
+
         if (errorMessage) {
             logData.error = errorMessage;
         }

From 29d1fb127497d36579afd1b292ee35ac285faec9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 5 Jun 2018 13:02:14 +0200
Subject: [PATCH 121/133] logging GzipSize

---
 app/services/stream_copy.js | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 3d92ddfd..5acfe077 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -110,11 +110,20 @@ module.exports = {
                         return cb(err);
                     }
                 })
-                .on('data', data => metrics.size += data.length)
+                .on('data', data => {
+                    if (gzip) {
+                        metrics.addGzipSize(data.length)
+                    } else {
+                        metrics.addSize(data.length)
+                    }
+                })
                 .on('end', () => requestEnded = true);
 
             if (gzip) {
-                req.pipe(zlib.createGunzip()).pipe(pgstream);
+                req
+                    .pipe(zlib.createGunzip())
+                    .on('data', data => metrics.addSize(data.length))
+                    .pipe(pgstream);
             } else {
                 req.pipe(pgstream);
             }

From a8ccacbc09644c6f7251c07675e2b9c5fc51b142 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 5 Jun 2018 13:16:01 +0200
Subject: [PATCH 122/133] size returns to unzipped

---
 test/acceptance/copy-endpoints.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 4a8ef9cc..5bea57ca 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -180,7 +180,7 @@ describe('copy-endpoints', function() {
             const headers = JSON.parse(res.headers['x-sqlapi-profiler']);
             assert.ok(headers.copyFrom);
             const metrics = headers.copyFrom;
-            assert.equal(metrics.size, 96);
+            assert.equal(metrics.size, 57);
             assert.equal(metrics.format, 'CSV');
             assert.equal(metrics.time, response.time);
             assert.equal(metrics.rows, response.total_rows);

From 66af518debb3d62a07e029b3e727821e1552e7b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Tue, 5 Jun 2018 13:16:14 +0200
Subject: [PATCH 123/133] jshint

---
 app/services/stream_copy.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/services/stream_copy.js b/app/services/stream_copy.js
index 5acfe077..f9bd2a70 100644
--- a/app/services/stream_copy.js
+++ b/app/services/stream_copy.js
@@ -112,9 +112,9 @@ module.exports = {
                 })
                 .on('data', data => {
                     if (gzip) {
-                        metrics.addGzipSize(data.length)
+                        metrics.addGzipSize(data.length);
                     } else {
-                        metrics.addSize(data.length)
+                        metrics.addSize(data.length);
                     }
                 })
                 .on('end', () => requestEnded = true);

From 2bcea0484ab08ac363cb280da5dcbfef0a60b0fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Fri, 8 Jun 2018 10:59:34 +0200
Subject: [PATCH 124/133] rename BunyanLogger to Logger

---
 app/controllers/copy_controller.js           | 4 ++--
 app/services/{bunyan_logger.js => logger.js} | 4 ++--
 batch/batch-logger.js                        | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)
 rename app/services/{bunyan_logger.js => logger.js} (92%)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 0bc652c1..1a723b3f 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -8,7 +8,7 @@ const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
 const { initializeProfilerMiddleware } = require('../middlewares/profiler');
 const rateLimitsMiddleware = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
-const BunyanLogger = require('../services/bunyan_logger');
+const Logger = require('../services/logger');
 const errorHandlerFactory = require('../services/error_handler_factory');
 const streamCopy = require('../services/stream_copy');
 
@@ -18,7 +18,7 @@ function CopyController(metadataBackend, userDatabaseService, userLimitsService,
     this.userLimitsService = userLimitsService;
     this.statsClient = statsClient;
 
-    this.logger = new BunyanLogger(global.settings.dataIngestionLogPath, 'data-ingestion');
+    this.logger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion');
 }
 
 CopyController.prototype.route = function (app) {
diff --git a/app/services/bunyan_logger.js b/app/services/logger.js
similarity index 92%
rename from app/services/bunyan_logger.js
rename to app/services/logger.js
index a0ab8920..87b6cba1 100644
--- a/app/services/bunyan_logger.js
+++ b/app/services/logger.js
@@ -2,7 +2,7 @@
 
 const bunyan = require('bunyan');
 
-class BunyanLogger {
+class Logger {
     constructor (path, name) {
         const stream = {
             level: process.env.NODE_ENV === 'test' ? 'fatal' : 'info'
@@ -30,4 +30,4 @@ class BunyanLogger {
     }
 }
  
-module.exports = BunyanLogger;
+module.exports = Logger;
diff --git a/batch/batch-logger.js b/batch/batch-logger.js
index 67b7aafc..ef132bd2 100644
--- a/batch/batch-logger.js
+++ b/batch/batch-logger.js
@@ -1,8 +1,8 @@
 'use strict';
 
-const BunyanLogger = require('../app/services/bunyan_logger');
+const Logger = require('../app/services/logger');
 
-class BatchLogger extends BunyanLogger {
+class BatchLogger extends Logger {
     constructor (path, name) {
         super(path, name);
     }

From b9a0fa78d283cc548f443fccf868da71d782d894 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20Mart=C3=ADn?= <oleurud@gmail.com>
Date: Mon, 11 Jun 2018 14:55:37 +0200
Subject: [PATCH 125/133] adding metadataBackend to user middleware in
 copycontroller

---
 app/controllers/copy_controller.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js
index 1a723b3f..1bbc8bd9 100644
--- a/app/controllers/copy_controller.js
+++ b/app/controllers/copy_controller.js
@@ -27,7 +27,7 @@ CopyController.prototype.route = function (app) {
     const copyFromMiddlewares = endpointGroup => {
         return [
             initializeProfilerMiddleware('copyfrom'),
-            userMiddleware(),
+            userMiddleware(this.metadataBackend),
             rateLimitsMiddleware(this.userLimitsService, endpointGroup),
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),
@@ -42,7 +42,7 @@ CopyController.prototype.route = function (app) {
     const copyToMiddlewares = endpointGroup => {
         return [
             initializeProfilerMiddleware('copyto'),
-            userMiddleware(),
+            userMiddleware(this.metadataBackend),
             rateLimitsMiddleware(this.userLimitsService, endpointGroup),
             authorizationMiddleware(this.metadataBackend),
             connectionParamsMiddleware(this.userDatabaseService),

From bbe5db64d02332a22ebed6c6bef42298cf7db46d Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 11 Jun 2018 14:59:56 +0200
Subject: [PATCH 126/133] Use our patched node-pg-copy-streams

The patch includes a performance improvement for the COPY TO. It can
eventually be removed in the PR to upstream is approved. See

https://github.com/brianc/node-pg-copy-streams/pull/70

and

https://github.com/CartoDB/node-pg-copy-streams/pull/1
---
 npm-shrinkwrap.json | 4 ++--
 package.json        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index 1083e6ab..de663306 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -853,8 +853,8 @@
     },
     "pg-copy-streams": {
       "version": "1.2.0",
-      "from": "pg-copy-streams@>=1.2.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-1.2.0.tgz"
+      "from": "cartodb/node-pg-copy-streams#v1.2.0-carto.1",
+      "resolved": "git://github.com/cartodb/node-pg-copy-streams.git#d7e5c1383ff9b43890f2c37bc38cfed1afc2523f"
     },
     "pg-int8": {
       "version": "1.0.1",
diff --git a/package.json b/package.json
index d9fd61e0..c6591248 100644
--- a/package.json
+++ b/package.json
@@ -32,7 +32,7 @@
     "node-statsd": "~0.0.7",
     "node-uuid": "^1.4.7",
     "oauth-client": "0.3.0",
-    "pg-copy-streams": "^1.2.0",
+    "pg-copy-streams": "cartodb/node-pg-copy-streams#v1.2.0-carto.1",
     "qs": "~6.2.1",
     "queue-async": "~1.0.7",
     "redis-mpool": "0.5.0",

From 52590faeac69209b1477ddadcd1f02e6b5fb8f61 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Fri, 8 Jun 2018 10:53:06 +0200
Subject: [PATCH 127/133] Add a test for client disconnection scenarios

---
 test/acceptance/copy-endpoints.js | 53 +++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 5bea57ca..db140ae7 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -330,3 +330,56 @@ describe('copy-endpoints db connections', function() {
         });
     });
 });
+
+describe('copy-endpoints client disconnection', function() {
+    before(function() {
+        this.db_pool_size = global.settings.db_pool_size;
+        global.settings.db_pool_size = 1;
+    });
+
+    after(function() {
+        global.settings.db_pool_size = this.db_pool_size;
+    });
+
+    it('copy to returns the connection to the pool if the client disconnects', function(done) {
+        const request = require('request');
+        const port = 5555;
+        const host = '127.0.0.1';
+
+        var listen = function() {
+            var listener = server.listen(port, host);
+            listener.on('listening', onServerListening);
+        };
+
+        var onServerListening = function() {
+            const requestParams = {
+                url: 'http://' + host + ':' + port + '/api/v1/sql/copyto?' + querystring.stringify({
+                    q: 'COPY populated_places_simple_reduced TO STDOUT',
+                }),
+                headers: { host: 'vizzuality.cartodb.com' },
+                method: 'GET',
+                timeout: 1
+            };
+            request(requestParams, function(err, response, body) {
+                // we're expecting a timeout error
+                assert.ok(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT');
+
+                // check now that we can perform a regular query, cause the connection should be back to the pool
+                request({
+                    url: 'http://' + host + ':' + port + '/api/v1/sql?' + querystring.stringify({
+                        q: 'SELECT 1',
+                    }),
+                    headers: { host: 'vizzuality.cartodb.com' },
+                    method: 'GET',
+                    timeout: 5000
+                }, function(err, response, body) {
+                    assert.ifError(err);
+                    assert(response.statusCode == 200);
+                    done();
+                });
+            });
+        }
+
+        listen();
+    });
+});

From 4b6ee133dfef78b2f30906e24fe59699dbf18e30 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 11 Jun 2018 18:18:44 +0200
Subject: [PATCH 128/133] Mess a little with the timeouts

---
 test/acceptance/copy-endpoints.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index db140ae7..6eaa8c32 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -334,11 +334,14 @@ describe('copy-endpoints db connections', function() {
 describe('copy-endpoints client disconnection', function() {
     before(function() {
         this.db_pool_size = global.settings.db_pool_size;
+        this.node_socket_timeout = global.settings.node_socket_timeout;
         global.settings.db_pool_size = 1;
+        global.settings.node_socket_timeout = 100;
     });
 
     after(function() {
         global.settings.db_pool_size = this.db_pool_size;
+        global.settings.node_socket_timeout = this.node_socket_timeout;
     });
 
     it('copy to returns the connection to the pool if the client disconnects', function(done) {

From 62fe8abd0a2417b25e52be01e2af08433693c553 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 11 Jun 2018 18:38:18 +0200
Subject: [PATCH 129/133] A much cleaner test

---
 test/acceptance/copy-endpoints.js | 56 ++++++++++++-------------------
 1 file changed, 22 insertions(+), 34 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 6eaa8c32..c4d08cd7 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -345,44 +345,32 @@ describe('copy-endpoints client disconnection', function() {
     });
 
     it('copy to returns the connection to the pool if the client disconnects', function(done) {
-        const request = require('request');
-        const port = 5555;
-        const host = '127.0.0.1';
 
-        var listen = function() {
-            var listener = server.listen(port, host);
-            listener.on('listening', onServerListening);
-        };
-
-        var onServerListening = function() {
-            const requestParams = {
-                url: 'http://' + host + ':' + port + '/api/v1/sql/copyto?' + querystring.stringify({
-                    q: 'COPY populated_places_simple_reduced TO STDOUT',
+        var assertCanReuseConnection = function () {
+            assert.response(server, {
+                url: '/api/v1/sql?' + querystring.stringify({
+                    q: 'SELECT 1',
                 }),
                 headers: { host: 'vizzuality.cartodb.com' },
-                method: 'GET',
-                timeout: 1
-            };
-            request(requestParams, function(err, response, body) {
-                // we're expecting a timeout error
-                assert.ok(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT');
-
-                // check now that we can perform a regular query, cause the connection should be back to the pool
-                request({
-                    url: 'http://' + host + ':' + port + '/api/v1/sql?' + querystring.stringify({
-                        q: 'SELECT 1',
-                    }),
-                    headers: { host: 'vizzuality.cartodb.com' },
-                    method: 'GET',
-                    timeout: 5000
-                }, function(err, response, body) {
-                    assert.ifError(err);
-                    assert(response.statusCode == 200);
-                    done();
-                });
+                method: 'GET'
+            }, {}, function(err, res) {
+                assert.ifError(err);
+                assert.ok(res.statusCode === 200);
+                done();
             });
-        }
+        };
 
-        listen();
+        assert.response(server, {
+            url: '/api/v1/sql/copyto?' + querystring.stringify({
+                q: 'COPY populated_places_simple_reduced TO STDOUT',
+            }),
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET',
+            timeout: 1
+        }, {}, function(err, res) {
+            // we're expecting a timeout error
+            assert.ok(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT');
+            assertCanReuseConnection();
+        });
     });
 });

From 1417dedfd31c1ffa8ee1390f456b982e72944708 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 11 Jun 2018 18:45:12 +0200
Subject: [PATCH 130/133] Final touches for client disconnect test

---
 test/acceptance/copy-endpoints.js | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index c4d08cd7..cad63057 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -334,19 +334,16 @@ describe('copy-endpoints db connections', function() {
 describe('copy-endpoints client disconnection', function() {
     before(function() {
         this.db_pool_size = global.settings.db_pool_size;
-        this.node_socket_timeout = global.settings.node_socket_timeout;
         global.settings.db_pool_size = 1;
-        global.settings.node_socket_timeout = 100;
     });
 
     after(function() {
         global.settings.db_pool_size = this.db_pool_size;
-        global.settings.node_socket_timeout = this.node_socket_timeout;
     });
 
     it('copy to returns the connection to the pool if the client disconnects', function(done) {
 
-        var assertCanReuseConnection = function () {
+        var assertCanReuseConnection = function (done) {
             assert.response(server, {
                 url: '/api/v1/sql?' + querystring.stringify({
                     q: 'SELECT 1',
@@ -366,11 +363,11 @@ describe('copy-endpoints client disconnection', function() {
             }),
             headers: { host: 'vizzuality.cartodb.com' },
             method: 'GET',
-            timeout: 1
+            timeout: 5
         }, {}, function(err, res) {
             // we're expecting a timeout error
             assert.ok(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT');
-            assertCanReuseConnection();
+            assertCanReuseConnection(done);
         });
     });
 });

From 66664de2867e8eb2fcd87bee44a4f2794dacb154 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 11 Jun 2018 19:04:33 +0200
Subject: [PATCH 131/133] Make the test far more robust

---
 test/acceptance/copy-endpoints.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index cad63057..27e7de02 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -359,13 +359,14 @@ describe('copy-endpoints client disconnection', function() {
 
         assert.response(server, {
             url: '/api/v1/sql/copyto?' + querystring.stringify({
-                q: 'COPY populated_places_simple_reduced TO STDOUT',
+                q: 'COPY (SELECT * FROM generate_series(1, 100000)) TO STDOUT',
             }),
             headers: { host: 'vizzuality.cartodb.com' },
             method: 'GET',
-            timeout: 5
+            timeout: 10
         }, {}, function(err, res) {
             // we're expecting a timeout error
+            assert.ok(err);
             assert.ok(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT');
             assertCanReuseConnection(done);
         });

From b1073dc401f38a87b0ee6d15276cf57bbc3cae94 Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 11 Jun 2018 19:17:23 +0200
Subject: [PATCH 132/133] Test for the COPY FROM + disconnect

---
 test/acceptance/copy-endpoints.js | 54 +++++++++++++++++++++----------
 1 file changed, 37 insertions(+), 17 deletions(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 27e7de02..03ab6534 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -332,6 +332,9 @@ describe('copy-endpoints db connections', function() {
 });
 
 describe('copy-endpoints client disconnection', function() {
+    // Give it enough time to connect and issue the query but not too much so as to disconnect in the middle of the query
+    const client_disconnect_timeout = 10;
+
     before(function() {
         this.db_pool_size = global.settings.db_pool_size;
         global.settings.db_pool_size = 1;
@@ -341,34 +344,51 @@ describe('copy-endpoints client disconnection', function() {
         global.settings.db_pool_size = this.db_pool_size;
     });
 
-    it('copy to returns the connection to the pool if the client disconnects', function(done) {
-
-        var assertCanReuseConnection = function (done) {
-            assert.response(server, {
-                url: '/api/v1/sql?' + querystring.stringify({
-                    q: 'SELECT 1',
-                }),
-                headers: { host: 'vizzuality.cartodb.com' },
-                method: 'GET'
-            }, {}, function(err, res) {
-                assert.ifError(err);
-                assert.ok(res.statusCode === 200);
-                done();
-            });
-        };
+    var assertCanReuseConnection = function (done) {
+        assert.response(server, {
+            url: '/api/v1/sql?' + querystring.stringify({
+                q: 'SELECT 1',
+            }),
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        }, {}, function(err, res) {
+            assert.ifError(err);
+            assert.ok(res.statusCode === 200);
+            done();
+        });
+    };
 
+    it('COPY TO returns the connection to the pool if the client disconnects', function(done) {
         assert.response(server, {
             url: '/api/v1/sql/copyto?' + querystring.stringify({
                 q: 'COPY (SELECT * FROM generate_series(1, 100000)) TO STDOUT',
             }),
             headers: { host: 'vizzuality.cartodb.com' },
             method: 'GET',
-            timeout: 10
-        }, {}, function(err, res) {
+            timeout: client_disconnect_timeout
+        }, {}, function(err) {
             // we're expecting a timeout error
             assert.ok(err);
             assert.ok(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT');
             assertCanReuseConnection(done);
         });
     });
+
+    it('COPY FROM returns the connection to the pool if the client disconnects', function(done) {
+        assert.response(server, {
+            url: '/api/v1/sql/copyfrom?' + querystring.stringify({
+                q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+            }),
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'POST',
+            data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+            timeout: client_disconnect_timeout
+        }, {}, function(err) {
+            // we're expecting a timeout error
+            assert.ok(err);
+            assert.ok(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT');
+            assertCanReuseConnection(done);
+        });
+    });
+
 });

From fc12917938a51606cd2fda918cc67535ccb1d04b Mon Sep 17 00:00:00 2001
From: Rafa de la Torre <rtorre@carto.com>
Date: Mon, 11 Jun 2018 19:26:40 +0200
Subject: [PATCH 133/133] Please jshint

---
 test/acceptance/copy-endpoints.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints.js
index 03ab6534..a6bfdb2e 100644
--- a/test/acceptance/copy-endpoints.js
+++ b/test/acceptance/copy-endpoints.js
@@ -332,7 +332,8 @@ describe('copy-endpoints db connections', function() {
 });
 
 describe('copy-endpoints client disconnection', function() {
-    // Give it enough time to connect and issue the query but not too much so as to disconnect in the middle of the query
+    // Give it enough time to connect and issue the query
+    // but not too much so as to disconnect in the middle of the query.
     const client_disconnect_timeout = 10;
 
     before(function() {