Add log-collector utlity, it will be moved to its onw repository. Attaching it here fro development purposes. Try it with the following command LOG_LEVEL=info npm t | node ./log-collector.js

log-collector.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const split = require('split2');
+const assingDeep = require('assign-deep');
+const logs = new Map();
+const { Transform } = require('readable-stream');
+
+const LEVELS = {
+    10: 'trace',
+    20: 'debug',
+    30: 'info',
+    40: 'warn',
+    50: 'error',
+    60: 'fatal'
+}
+
+function logTransport () {
+    return new Transform({
+        transform: function transform (chunk, enc, callback) {
+            let entry;
+
+            try {
+                entry = JSON.parse(chunk);
+            } catch (error) {
+                // this.push(chunk + '\n');
+                return callback();
+            }
+
+            const { id, end } = entry;
+
+            if (id === undefined) {
+                entry.level = LEVELS[entry.level];
+                this.push(`${JSON.stringify(entry)}\n`);
+                return callback();
+            }
+
+            if (end === true) {
+                const accEntry = logs.get(id);
+                accEntry.level = LEVELS[accEntry.level];
+                accEntry.time = entry.time;
+                this.push(`${JSON.stringify(accEntry)}\n`);
+                logs.delete(id);
+                return callback();
+            }
+
+            if (logs.has(id)) {
+                const accEntry = logs.get(id);
+
+                if (accEntry.level > entry.level) {
+                    delete entry.level
+                }
+
+                let error;
+                if (Object.prototype.hasOwnProperty.call(accEntry, 'error') && Object.prototype.hasOwnProperty.call(entry, 'error')) {
+                    logs.set(id, assingDeep({}, accEntry, entry, { error: accEntry.error.concat(entry.error) }));
+                } else {
+                    logs.set(id, assingDeep({}, accEntry, entry));
+                }
+            } else {
+                logs.set(id, entry);
+            }
+
+            callback();
+        }
+    })
+}
+
+process.stdin
+    .pipe(split())
+    .pipe(logTransport())
+    .pipe(process.stdout);

package-lock.json

@@ -633,6 +633,19 @@
       "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
       "dev": true
     },
+    "assign-deep": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/assign-deep/-/assign-deep-1.0.1.tgz",
+      "integrity": "sha512-CSXAX79mibneEYfqLT5FEmkqR5WXF+xDRjgQQuVf6wSCXCYU8/vHttPidNar7wJ5BFmKAo8Wei0rCtzb+M/yeA==",
+      "requires": {
+        "assign-symbols": "^2.0.2"
+      }
+    },
+    "assign-symbols": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-2.0.2.tgz",
+      "integrity": "sha512-9sBQUQZMKFKcO/C3Bo6Rx4CQany0R0UeVcefNGRRdW2vbmaMOhV1sbmlXcQLcD56juLXbSGTBm0GGuvmrAF8pA=="
+    },
     "astral-regex": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz",
@@ -5530,7 +5543,6 @@
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
       "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-      "dev": true,
       "requires": {
         "inherits": "^2.0.3",
         "string_decoder": "^1.1.1",
@@ -5970,7 +5982,6 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/split2/-/split2-3.1.1.tgz",
       "integrity": "sha512-emNzr1s7ruq4N+1993yht631/JH+jaj0NYBosuKmLcq+JkGQ9MmTw1RB1fGaTCzUuseRIClrlSLHRNYGwWQ58Q==",
-      "dev": true,
       "requires": {
         "readable-stream": "^3.0.0"
       }
@@ -6251,7 +6262,6 @@
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
       "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
-      "dev": true,
       "requires": {
         "safe-buffer": "~5.2.0"
       },
@@ -6259,8 +6269,7 @@
         "safe-buffer": {
           "version": "5.2.1",
           "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
-          "dev": true
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
         }
       }
     },

package.json

@@ -36,6 +36,7 @@
   "dependencies": {
     "@carto/fqdn-sync": "0.2.2",
     "@google-cloud/pubsub": "1.5.0",
+    "assign-deep": "^1.0.1",
     "basic-auth": "2.0.0",
     "body-parser": "1.18.3",
     "camshaft": "github:cartodb/camshaft#dgaubert/ch78389/camshaft-replace-logger-from-bunyan-to-pino",
@@ -56,6 +57,7 @@
     "redis-mpool": "^0.8.0",
     "request": "2.87.0",
     "semver": "5.5.0",
+    "split2": "^3.1.1",
     "step-profiler": "0.3.0",
     "turbo-carto": "0.21.2",
     "underscore": "1.6.0",