762a240890
- Log system revamp: - Logs to stdout, disabled while testing - Use header `X-Request-Id`, or create a new `uuid` when no present, to identyfy log entries - Be able to set log level from env variable `LOG_LEVEL`, useful while testing: `LOG_LEVEL=info npm test`; even more human-readable: `LOG_LEVEL=info npm t | ./node_modules/.bin/pino-pretty` - Be able to reduce the footprint in the final log file depending on the environment - Use one logger for every service: Queries, Batch Queries (Jobs), and Data Ingestion (CopyTo/CopyFrom) - Stop using headers such as: `X-SQL-API-Log`, `X-SQL-API-Profiler`, and `X-SQL-API-Errors` as a way to log info. - Be able to tag requests with labels as an easier way to provide business metrics - Metro: Add log-collector utility (`metro`), it will be moved to its own repository. Attaching it here fro development purposes. Try it with the following command `LOG_LEVEL=info npm t | node metro` - Metro: Creates `metrics-collector.js` a stream to update Prometheus' counters and histograms and exposes them via Express' app (`:9145/metrics`). Use the ones defined in `grok_exporter` Announcements: - Profiler is always set. No need to check its existence anymore - Unify profiler usage for every endpoint Bug fixes: - Avoid hung requests while fetching user identifier
81 lines
2.8 KiB
JavaScript
81 lines
2.8 KiB
JavaScript
'use strict';
|
|
|
|
require('../../helper');
|
|
|
|
var BATCH_SOURCE = '../../../lib/batch/';
|
|
|
|
var assert = require('../../support/assert');
|
|
var redisUtils = require('../../support/redis-utils');
|
|
|
|
var Logger = require('../../../lib/utils/logger');
|
|
var JobQueue = require(BATCH_SOURCE + 'job-queue');
|
|
var JobBackend = require(BATCH_SOURCE + 'job-backend');
|
|
var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher');
|
|
var jobStatus = require(BATCH_SOURCE + 'job-status');
|
|
var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user-database-metadata-service');
|
|
var JobCanceller = require(BATCH_SOURCE + 'job-canceller');
|
|
var JobService = require(BATCH_SOURCE + 'job-service');
|
|
var JobRunner = require(BATCH_SOURCE + 'job-runner');
|
|
var QueryRunner = require(BATCH_SOURCE + 'query-runner');
|
|
|
|
var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
|
|
var logger = new Logger();
|
|
var jobPublisher = new JobPublisher(redisUtils.getPool());
|
|
var jobQueue = new JobQueue(metadataBackend, jobPublisher, logger);
|
|
var jobBackend = new JobBackend(metadataBackend, jobQueue, logger);
|
|
var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend);
|
|
var jobCanceller = new JobCanceller();
|
|
var jobService = new JobService(jobBackend, jobCanceller, logger);
|
|
var queryRunner = new QueryRunner(userDatabaseMetadataService, logger);
|
|
var StatsD = require('node-statsd').StatsD;
|
|
var statsdClient = new StatsD(global.settings.statsd);
|
|
|
|
var USER = 'vizzuality';
|
|
var QUERY = 'select pg_sleep(0)';
|
|
var HOST = 'localhost';
|
|
var JOB = {
|
|
user: USER,
|
|
query: QUERY,
|
|
host: HOST,
|
|
dbname: 'cartodb_test_user_1_db',
|
|
dbuser: 'test_cartodb_user_1',
|
|
port: 5432,
|
|
pass: 'test_cartodb_user_1_pass'
|
|
};
|
|
|
|
describe('job runner', function () {
|
|
var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient);
|
|
|
|
after(function (done) {
|
|
redisUtils.clean('batch:*', function () {
|
|
redisUtils.clean('limits:batch:*', done);
|
|
});
|
|
});
|
|
|
|
it('.run() should run a job', function (done) {
|
|
jobService.create(JOB, function (err, job) {
|
|
if (err) {
|
|
return done(err);
|
|
}
|
|
|
|
jobRunner.run(job.data.job_id, function (err, job) {
|
|
if (err) {
|
|
return done(err);
|
|
}
|
|
|
|
assert.strictEqual(job.data.status, jobStatus.DONE);
|
|
done();
|
|
});
|
|
});
|
|
});
|
|
|
|
it('.run() should return a job not found error', function (done) {
|
|
jobRunner.run('wadus_job_id', function (err) {
|
|
assert.ok(err, err);
|
|
assert.strictEqual(err.name, 'NotFoundError');
|
|
assert.strictEqual(err.message, 'Job with id wadus_job_id not found');
|
|
done();
|
|
});
|
|
});
|
|
});
|