CartoDB-SQL-API/lib/batch/models/job-multiple.js
Daniel García Aubert 762a240890 Breaking changes:
- Log system revamp:
  - Logs to stdout, disabled while testing
  - Use header `X-Request-Id`, or create a new `uuid` when no present, to identyfy log entries
  - Be able to set log level from env variable `LOG_LEVEL`, useful while testing: `LOG_LEVEL=info npm test`; even more human-readable: `LOG_LEVEL=info npm t | ./node_modules/.bin/pino-pretty`
  - Be able to reduce the footprint in the final log file depending on the environment
  - Use one logger for every service: Queries, Batch Queries (Jobs), and Data Ingestion (CopyTo/CopyFrom)
  - Stop using headers such as: `X-SQL-API-Log`, `X-SQL-API-Profiler`, and `X-SQL-API-Errors` as a way to log info.
  - Be able to tag requests with labels as an easier way to provide business metrics
  - Metro: Add log-collector utility (`metro`), it will be moved to its own repository. Attaching it here fro development purposes. Try it with the following command `LOG_LEVEL=info npm t | node metro`
  - Metro: Creates `metrics-collector.js` a stream to update Prometheus' counters and histograms and exposes them via Express' app (`:9145/metrics`). Use the ones defined in `grok_exporter`

Announcements:
- Profiler is always set. No need to check its existence anymore
- Unify profiler usage for every endpoint

Bug fixes:
- Avoid hung requests while fetching user identifier
2020-06-30 17:42:59 +02:00

122 lines
3.5 KiB
JavaScript

'use strict';
var util = require('util');
var JobBase = require('./job-base');
var jobStatus = require('../job-status');
function JobMultiple (jobDefinition) {
JobBase.call(this, jobDefinition);
this.init();
}
util.inherits(JobMultiple, JobBase);
module.exports = JobMultiple;
JobMultiple.is = function (query) {
if (!Array.isArray(query)) {
return false;
}
// 1. From user: ['select * from ...', 'select * from ...']
// 2. From redis: [ { query: 'select * from ...', status: 'pending' },
// { query: 'select * from ...', status: 'pending' } ]
for (var i = 0; i < query.length; i++) {
if (typeof query[i] !== 'string') {
if (typeof query[i].query !== 'string') {
return false;
}
}
}
return true;
};
JobMultiple.prototype.init = function () {
if (!this.data.status) {
this.data.status = jobStatus.PENDING;
}
for (var i = 0; i < this.data.query.length; i++) {
if (!this.data.query[i].query && !this.data.query[i].status) {
this.data.query[i] = {
query: this.data.query[i],
status: jobStatus.PENDING
};
}
}
};
JobMultiple.prototype.getNextQuery = function () {
for (var i = 0; i < this.data.query.length; i++) {
if (this.data.query[i].status === jobStatus.PENDING) {
return this.data.query[i].query;
}
}
};
JobMultiple.prototype.setQuery = function (query) {
if (!JobMultiple.is(query)) {
throw new Error('You must indicate a valid SQL');
}
JobMultiple.super_.prototype.setQuery.call(this, query);
};
JobMultiple.prototype.setStatus = function (finalStatus, errorMesssage) {
var initialStatus = this.data.status;
// if transition is to "done" and there are more queries to run
// then job status must be "pending" instead of "done"
// else job status transition to done (if "running")
if (finalStatus === jobStatus.DONE && this.hasNextQuery()) {
JobMultiple.super_.prototype.setStatus.call(this, jobStatus.PENDING);
} else {
JobMultiple.super_.prototype.setStatus.call(this, finalStatus, errorMesssage);
}
for (var i = 0; i < this.data.query.length; i++) {
var isValid = JobMultiple.super_.prototype.isValidTransition(this.data.query[i].status, finalStatus);
if (isValid) {
this.data.query[i].status = finalStatus;
if (finalStatus === jobStatus.FAILED && errorMesssage) {
this.data.query[i].failed_reason = errorMesssage;
}
return;
}
}
throw new Error('Cannot set status from ' + initialStatus + ' to ' + finalStatus);
};
JobMultiple.prototype.toJSON = function () {
const queries = this.data.query;
return {
type: this.constructor.name,
id: this.data.job_id,
username: this.data.user,
status: this.data.status,
created: this.data.created_at,
updated: this.data.updated_at,
elapsed: elapsedTime(this.data.created_at, this.data.updated_at),
dbhost: this.data.host,
queries: queries.map((query) => {
return {
status: query.status,
failed_reason: query.failed_reason
};
})
};
};
function elapsedTime (startedAt, endedAt) {
if (!startedAt || !endedAt) {
return;
}
var start = new Date(startedAt);
var end = new Date(endedAt);
return end.getTime() - start.getTime();
}