55 lines
1.7 KiB
JavaScript
55 lines
1.7 KiB
JavaScript
import { Queue, Worker } from 'bullmq';
|
|
import Redis from 'ioredis';
|
|
import log4js from './logger.js';
|
|
import { fileQueue } from './file.worker.js';
|
|
import { bodyQueue } from './body.worker.js';
|
|
|
|
const logger = log4js.getLogger('header');
|
|
|
|
const connection = {
|
|
host: process.env.REDIS_HOST || 'localhost',
|
|
port: process.env.REDIS_PORT || 6379,
|
|
};
|
|
|
|
const redis = new Redis(connection);
|
|
|
|
export const headerQueue = new Queue('header-queue', { connection });
|
|
|
|
const SUBJECT_REGEX = /"(.+)"(?: yEnc)? \((\d+)\/(\d+)\)/;
|
|
|
|
export const startHeaderWorker = () => {
|
|
const headerWorker = new Worker('header-queue', async job => {
|
|
const header = job.data;
|
|
const subject = header.subject;
|
|
|
|
const match = subject.match(SUBJECT_REGEX);
|
|
|
|
if (match) {
|
|
const filename = match[1];
|
|
const part = parseInt(match[2], 10);
|
|
const total = parseInt(match[3], 10);
|
|
|
|
const fileKey = `file:${filename}`;
|
|
await redis.hset(fileKey, part, JSON.stringify(header));
|
|
|
|
const partCount = await redis.hlen(fileKey);
|
|
|
|
if (partCount === total) {
|
|
const fileParts = await redis.hgetall(fileKey);
|
|
await fileQueue.add('process-file', { filename, parts: fileParts });
|
|
await redis.del(fileKey);
|
|
logger.info(`File "${filename}" is complete and moved to file-queue.`, fileParts);
|
|
} else {
|
|
logger.info(`Stored part ${part}/${total} for file "${filename}"`);
|
|
}
|
|
} else {
|
|
logger.warn(`Could not parse subject: "${subject}". Moving to body-queue.`);
|
|
await bodyQueue.add('process-body', { header });
|
|
}
|
|
}, { connection });
|
|
|
|
headerWorker.on('failed', (job, err) => {
|
|
logger.error(`Header job ${job.id} failed with error: ${err.message}`);
|
|
});
|
|
};
|