66 lines
2.3 KiB
TypeScript
66 lines
2.3 KiB
TypeScript
import { BaseCommand } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import { Worker } from 'bullmq'
|
|
import queueConfig from '#config/queue'
|
|
import QueueService from '#services/QueueService'
|
|
import RedisService from '#services/RedisService'
|
|
|
|
export default class HeaderWorker extends BaseCommand {
|
|
public static commandName = 'worker:header'
|
|
public static description = 'Starts a worker to process headers.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
public async run() {
|
|
this.logger.info('Starting header worker...')
|
|
|
|
const redis = RedisService.client
|
|
const fileQueue = QueueService.fileQueue
|
|
const bodyQueue = QueueService.bodyQueue
|
|
|
|
const worker = new Worker('header-queue', async (job) => {
|
|
const { header, group } = job.data
|
|
if (!header || !header.subject) {
|
|
this.logger.warning(`Received job with invalid header data. JobID: ${job.id}`)
|
|
return
|
|
}
|
|
|
|
const subject = header.subject
|
|
const SUBJECT_REGEX = /"(.+)"(?: yEnc)? \((\d+)\/(\d+)\)/
|
|
const match = subject.match(SUBJECT_REGEX)
|
|
|
|
if (match) {
|
|
const filename = match[1]
|
|
const part = parseInt(match[2], 10)
|
|
const total = parseInt(match[3], 10)
|
|
|
|
const fileKey = `file:${filename}`
|
|
await redis.hset(fileKey, part, JSON.stringify(header))
|
|
|
|
const partCount = await redis.hlen(fileKey)
|
|
|
|
if (partCount === total) {
|
|
const fileParts = await redis.hgetall(fileKey)
|
|
await fileQueue.add('process-file', { filename, parts: fileParts, groups: [group] })
|
|
await redis.del(fileKey)
|
|
this.logger.info(`File "${filename}" is complete and moved to file-queue.`)
|
|
} else {
|
|
this.logger.info(`Stored part ${part}/${total} for file "${filename}"`)
|
|
}
|
|
} else {
|
|
this.logger.warning(`Could not parse subject: "${subject}". Moving to body-queue.`)
|
|
await bodyQueue.add('process-body', { header, group })
|
|
}
|
|
}, { connection: queueConfig.connection })
|
|
|
|
worker.on('failed', (job, err) => {
|
|
this.logger.error(`Header job ${job?.id} failed: ${err.message}`)
|
|
})
|
|
|
|
this.logger.info('Header worker started and listening for jobs.')
|
|
await new Promise(() => {})
|
|
}
|
|
}
|