diff --git a/log-collector.js b/log-collector.js new file mode 100644 index 00000000..2b4c1f1c --- /dev/null +++ b/log-collector.js @@ -0,0 +1,71 @@ +'use strict' + +const split = require('split2'); +const assingDeep = require('assign-deep'); +const logs = new Map(); +const { Transform } = require('readable-stream'); + +const LEVELS = { + 10: 'trace', + 20: 'debug', + 30: 'info', + 40: 'warn', + 50: 'error', + 60: 'fatal' +} + +function logTransport () { + return new Transform({ + transform: function transform (chunk, enc, callback) { + let entry; + + try { + entry = JSON.parse(chunk); + } catch (error) { + // this.push(chunk + '\n'); + return callback(); + } + + const { id, end } = entry; + + if (id === undefined) { + entry.level = LEVELS[entry.level]; + this.push(`${JSON.stringify(entry)}\n`); + return callback(); + } + + if (end === true) { + const accEntry = logs.get(id); + accEntry.level = LEVELS[accEntry.level]; + accEntry.time = entry.time; + this.push(`${JSON.stringify(accEntry)}\n`); + logs.delete(id); + return callback(); + } + + if (logs.has(id)) { + const accEntry = logs.get(id); + + if (accEntry.level > entry.level) { + delete entry.level + } + + let error; + if (Object.prototype.hasOwnProperty.call(accEntry, 'error') && Object.prototype.hasOwnProperty.call(entry, 'error')) { + logs.set(id, assingDeep({}, accEntry, entry, { error: accEntry.error.concat(entry.error) })); + } else { + logs.set(id, assingDeep({}, accEntry, entry)); + } + } else { + logs.set(id, entry); + } + + callback(); + } + }) +} + +process.stdin + .pipe(split()) + .pipe(logTransport()) + .pipe(process.stdout); diff --git a/package-lock.json b/package-lock.json index 580ad405..6737c08d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -633,6 +633,19 @@ "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", "dev": true }, + "assign-deep": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/assign-deep/-/assign-deep-1.0.1.tgz", + "integrity": "sha512-CSXAX79mibneEYfqLT5FEmkqR5WXF+xDRjgQQuVf6wSCXCYU8/vHttPidNar7wJ5BFmKAo8Wei0rCtzb+M/yeA==", + "requires": { + "assign-symbols": "^2.0.2" + } + }, + "assign-symbols": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-2.0.2.tgz", + "integrity": "sha512-9sBQUQZMKFKcO/C3Bo6Rx4CQany0R0UeVcefNGRRdW2vbmaMOhV1sbmlXcQLcD56juLXbSGTBm0GGuvmrAF8pA==" + }, "astral-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", @@ -5530,7 +5543,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dev": true, "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -5970,7 +5982,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/split2/-/split2-3.1.1.tgz", "integrity": "sha512-emNzr1s7ruq4N+1993yht631/JH+jaj0NYBosuKmLcq+JkGQ9MmTw1RB1fGaTCzUuseRIClrlSLHRNYGwWQ58Q==", - "dev": true, "requires": { "readable-stream": "^3.0.0" } @@ -6251,7 +6262,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "requires": { "safe-buffer": "~5.2.0" }, @@ -6259,8 +6269,7 @@ "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" } } }, diff --git a/package.json b/package.json index af37467f..b0766b8f 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "dependencies": { "@carto/fqdn-sync": "0.2.2", "@google-cloud/pubsub": "1.5.0", + "assign-deep": "^1.0.1", "basic-auth": "2.0.0", "body-parser": "1.18.3", "camshaft": "github:cartodb/camshaft#dgaubert/ch78389/camshaft-replace-logger-from-bunyan-to-pino", @@ -56,6 +57,7 @@ "redis-mpool": "^0.8.0", "request": "2.87.0", "semver": "5.5.0", + "split2": "^3.1.1", "step-profiler": "0.3.0", "turbo-carto": "0.21.2", "underscore": "1.6.0",