75 lines
2.8 KiB
TypeScript
75 lines
2.8 KiB
TypeScript
import { BaseCommand } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import { Worker } from 'bullmq'
|
|
import fs from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import queueConfig from '#config/queue'
|
|
import QueueService from '#services/QueueService'
|
|
import NntpService from '#services/NntpService'
|
|
import { parseYencMeta } from '#services/YencService'
|
|
|
|
export default class BodyWorker extends BaseCommand {
|
|
public static commandName = 'worker:body'
|
|
public static description = 'Starts a worker to process article bodies for yEnc metadata.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
public async run() {
|
|
this.logger.info('Starting body worker...')
|
|
|
|
const pool = NntpService
|
|
const headerQueue = QueueService.headerQueue
|
|
|
|
const worker = new Worker('body-queue', async (job) => {
|
|
const { header, group } = job.data
|
|
const messageId = header['message-id']
|
|
this.logger.debug(`Processing header with unparsable subject: ${header.subject}`)
|
|
|
|
let conn
|
|
try {
|
|
conn = await pool.acquire()
|
|
const bodyBuffer: Buffer = (await conn.body(messageId)).data
|
|
|
|
try {
|
|
const meta = parseYencMeta(bodyBuffer)
|
|
if (meta.header.name) {
|
|
const { name, part, total } = meta.header
|
|
const newSubject = `"${name}" yEnc (${part}/${total})`
|
|
header.subject = newSubject
|
|
|
|
this.logger.info(`Found yEnc metadata in body. New subject: ${newSubject}`)
|
|
await headerQueue.add('process-header', { header, group })
|
|
} else {
|
|
this.logger.warning(`Could not find yEnc metadata in body for header: ${header.subject}`)
|
|
}
|
|
} catch (parseError: any) {
|
|
this.logger.error(`Failed to parse yEnc data for message ID ${messageId}. Dumping buffer.`)
|
|
const debugDir = path.join(this.app.appRoot.pathname, 'debug')
|
|
await fs.mkdir(debugDir, { recursive: true })
|
|
const timestamp = new Date().toISOString().replace(/:/g, '-')
|
|
const dumpFile = path.join(debugDir, `body-error-${timestamp}-${messageId.replace(/[<>]/g, '')}.bin`)
|
|
await fs.writeFile(dumpFile, bodyBuffer)
|
|
this.logger.error(`Problematic body buffer saved to: ${dumpFile}`)
|
|
throw parseError
|
|
}
|
|
} catch (error: any) {
|
|
this.logger.error(`Error in body worker for message ID ${messageId}: ${error.message}`)
|
|
throw error
|
|
} finally {
|
|
if (conn) {
|
|
pool.release(conn)
|
|
}
|
|
}
|
|
}, { connection: queueConfig.connection })
|
|
|
|
worker.on('failed', (job, err) => {
|
|
this.logger.error(`Body job ${job?.id} failed: ${err.message}`)
|
|
})
|
|
|
|
this.logger.info('Body worker started and listening for jobs.')
|
|
await new Promise(() => {})
|
|
}
|
|
}
|