import { Queue, Worker } from 'bullmq'; import Redis from 'ioredis'; import log4js from './logger.js'; import { fileQueue } from './file.worker.js'; import { bodyQueue } from './body.worker.js'; const logger = log4js.getLogger('header'); const connection = { host: process.env.REDIS_HOST || 'localhost', port: process.env.REDIS_PORT || 6379, }; const redis = new Redis(connection); export const headerQueue = new Queue('header-queue', { connection }); const SUBJECT_REGEX = /"(.+)"(?: yEnc)? \((\d+)\/(\d+)\)/; export const startHeaderWorker = () => { const headerWorker = new Worker('header-queue', async job => { const header = job.data; const subject = header.subject; const match = subject.match(SUBJECT_REGEX); if (match) { const filename = match[1]; const part = parseInt(match[2], 10); const total = parseInt(match[3], 10); const fileKey = `file:${filename}`; await redis.hset(fileKey, part, JSON.stringify(header)); const partCount = await redis.hlen(fileKey); if (partCount === total) { const fileParts = await redis.hgetall(fileKey); await fileQueue.add('process-file', { filename, parts: fileParts }); await redis.del(fileKey); logger.info(`File "${filename}" is complete and moved to file-queue.`, fileParts); } else { logger.info(`Stored part ${part}/${total} for file "${filename}"`); } } else { logger.warn(`Could not parse subject: "${subject}". Moving to body-queue.`); await bodyQueue.add('process-body', { header }); } }, { connection }); headerWorker.on('failed', (job, err) => { logger.error(`Header job ${job.id} failed with error: ${err.message}`); }); };