762a240890
- Log system revamp: - Logs to stdout, disabled while testing - Use header `X-Request-Id`, or create a new `uuid` when no present, to identyfy log entries - Be able to set log level from env variable `LOG_LEVEL`, useful while testing: `LOG_LEVEL=info npm test`; even more human-readable: `LOG_LEVEL=info npm t | ./node_modules/.bin/pino-pretty` - Be able to reduce the footprint in the final log file depending on the environment - Use one logger for every service: Queries, Batch Queries (Jobs), and Data Ingestion (CopyTo/CopyFrom) - Stop using headers such as: `X-SQL-API-Log`, `X-SQL-API-Profiler`, and `X-SQL-API-Errors` as a way to log info. - Be able to tag requests with labels as an easier way to provide business metrics - Metro: Add log-collector utility (`metro`), it will be moved to its own repository. Attaching it here fro development purposes. Try it with the following command `LOG_LEVEL=info npm t | node metro` - Metro: Creates `metrics-collector.js` a stream to update Prometheus' counters and histograms and exposes them via Express' app (`:9145/metrics`). Use the ones defined in `grok_exporter` Announcements: - Profiler is always set. No need to check its existence anymore - Unify profiler usage for every endpoint Bug fixes: - Avoid hung requests while fetching user identifier
88 lines
2.0 KiB
JavaScript
88 lines
2.0 KiB
JavaScript
'use strict';
|
|
|
|
const { getFormatFromCopyQuery } = require('../utils/query-info');
|
|
|
|
module.exports = class StreamCopyMetrics {
|
|
constructor (logger, type, sql, user, isGzip = false) {
|
|
this.logger = logger;
|
|
|
|
this.type = type;
|
|
this.format = getFormatFromCopyQuery(sql);
|
|
this.sql = sql;
|
|
this.isGzip = isGzip;
|
|
this.username = user;
|
|
this.size = 0;
|
|
this.gzipSize = 0;
|
|
this.rows = 0;
|
|
|
|
this.startTime = new Date();
|
|
this.endTime = null;
|
|
this.time = null;
|
|
|
|
this.success = true;
|
|
this.error = null;
|
|
|
|
this.ended = false;
|
|
}
|
|
|
|
addSize (size) {
|
|
this.size += size;
|
|
}
|
|
|
|
addGzipSize (size) {
|
|
this.gzipSize += size;
|
|
}
|
|
|
|
end (rows = null, error = null) {
|
|
if (this.ended) {
|
|
return;
|
|
}
|
|
|
|
this.ended = true;
|
|
|
|
if (Number.isInteger(rows)) {
|
|
this.rows = rows;
|
|
}
|
|
|
|
if (error instanceof Error) {
|
|
this.error = error;
|
|
}
|
|
|
|
this.endTime = new Date();
|
|
this.time = (this.endTime.getTime() - this.startTime.getTime()) / 1000;
|
|
|
|
this._log(
|
|
this.startTime.toISOString(),
|
|
this.isGzip && this.gzipSize ? this.gzipSize : null,
|
|
this.error ? this.error.message : null
|
|
);
|
|
}
|
|
|
|
_log (timestamp, gzipSize = null, errorMessage = null) {
|
|
const logData = {
|
|
type: this.type,
|
|
format: this.format,
|
|
size: this.size,
|
|
rows: this.rows,
|
|
gzip: this.isGzip,
|
|
username: this.username,
|
|
time: this.time,
|
|
timestamp,
|
|
sql: this.sql
|
|
};
|
|
|
|
if (gzipSize) {
|
|
logData.gzipSize = gzipSize;
|
|
}
|
|
|
|
if (errorMessage) {
|
|
logData.error = errorMessage;
|
|
this.success = false;
|
|
}
|
|
|
|
logData.success = this.success;
|
|
|
|
this.logger.info({ ingestion: logData });
|
|
}
|
|
};
|