2022-02-10 01:14:52 +08:00
|
|
|
const Logger = require('./lib/utils/logger');
|
|
|
|
const config = require('./config');
|
|
|
|
const fs = require('fs');
|
|
|
|
const redis = require('redis');
|
|
|
|
const { Worker } = require('worker_threads')
|
|
|
|
|
|
|
|
const logger = new Logger('presAnn Master');
|
|
|
|
logger.info("Running export-annotations");
|
|
|
|
|
|
|
|
const kickOffCollectorWorker = (jobId) => {
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
const worker = new Worker('./workers/collector.js', { workerData: jobId });
|
|
|
|
worker.on('message', resolve);
|
|
|
|
worker.on('error', reject);
|
|
|
|
worker.on('exit', (code) => {
|
|
|
|
if (code !== 0)
|
|
|
|
reject(new Error(`PresAnn Collector Worker stopped with exit code ${code}`));
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
function sleep(ms) {
|
|
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
|
|
}
|
|
|
|
|
|
|
|
(async () => {
|
|
|
|
const client = redis.createClient({
|
|
|
|
host: config.redis.host,
|
|
|
|
port: config.redis.port,
|
|
|
|
password: config.redis.password
|
|
|
|
});
|
|
|
|
|
|
|
|
client.on('error', (err) => logger.info('Redis Client Error', err));
|
|
|
|
|
|
|
|
await client.connect();
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
await sleep(config.redis.interval);
|
|
|
|
|
2022-02-10 19:48:54 +08:00
|
|
|
let job = await client.LPOP(config.redis.channels.queue)
|
2022-02-10 01:14:52 +08:00
|
|
|
|
|
|
|
const exportJob = JSON.parse(job);
|
|
|
|
|
|
|
|
if(job != null) {
|
|
|
|
logger.info('Received job', job)
|
|
|
|
|
2022-02-13 04:03:07 +08:00
|
|
|
// Create folder in dropbox
|
2022-02-14 01:10:37 +08:00
|
|
|
let dropbox = `${config.shared.presAnnDropboxDir}/${exportJob.jobId}`
|
2022-02-13 04:03:07 +08:00
|
|
|
fs.mkdirSync(dropbox, { recursive: true })
|
|
|
|
|
2022-02-10 01:14:52 +08:00
|
|
|
// Drop job into dropbox as JSON
|
2022-02-14 01:10:37 +08:00
|
|
|
fs.writeFile(`${dropbox}/job`, job, function(err) {
|
2022-02-10 01:14:52 +08:00
|
|
|
if(err) { return logger.error(err); }
|
|
|
|
});
|
|
|
|
|
2022-02-13 04:03:07 +08:00
|
|
|
kickOffCollectorWorker(exportJob.jobId)
|
2022-02-10 01:14:52 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
})();
|