import { BaseCommand } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import { Worker } from 'bullmq' import db from '@adonisjs/lucid/services/db' import queueConfig from '#config/queue' import Shutdown from '#services/Shutdown' import QueueService from '#services/QueueService' export default class FileWorker extends BaseCommand { public static commandName = 'worker:file' public static description = 'Starts a worker that turns completed binaries into files.' public static options: CommandOptions = { startApp: true, } public async run() { this.logger.info('Starting file worker...') const collectionQueue = QueueService.collectionQueue const worker = new Worker('file-queue', async (job) => { const { binaryId } = job.data const binary = await db.from('binaries').where('id', binaryId).first() if (!binary) { this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`) return } if (binary.conflicted_at) { this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`) return } const parts = await db .from('parts') .where('binary_id', binaryId) .orderBy('number') .select('number', 'message_id', 'bytes') if (parts.length < binary.total_parts) { throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`) } const messageIds = Object.fromEntries( parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }]) ) // Already known as a file, e.g. added from a posted NZB before its own articles completed. const existing = await db .from('files') .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? '']) .first() if (existing) { await db.from('binaries').where('id', binaryId).update({ file_id: existing.id }) await dropCopiedParts(binary) this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`) return } // file_key is unique: a second completion of the same upload inserts nothing. const knex = db.connection().getWriteClient() const now = new Date() const [inserted] = await knex('files') .insert({ file_key: binary.file_key, filename: binary.name, poster: binary.poster, date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(), parts: parts.length, message_ids: JSON.stringify(messageIds), groups: JSON.stringify(binary.groups), release_id: binary.release_id, real_name: binary.real_name, size: parts.reduce((sum, part) => sum + Number(part.bytes), 0), created_at: now, updated_at: now, }) .onConflict('file_key') .ignore() .returning('id') const fileId = inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id await db.from('binaries').where('id', binaryId).update({ file_id: fileId }) await dropCopiedParts(binary) if (!inserted) { this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`) return } this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`) await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` }) }, { connection: queueConfig.connection }) worker.on('failed', (job, err) => { this.logger.error(`File job ${job?.id} failed: ${err.message}`) }) this.logger.info('File worker started and listening for jobs.') await Shutdown.wait() this.logger.info('Shutting down: finishing active jobs...') await worker.close() await QueueService.closeAll() } } /** * The file holds every message id now. Binaries identified by their subject * (no size: header-derived) don't need their parts any more; body-derived * ones keep them, so articles of an equal-sized next volume are recognised * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17). */ async function dropCopiedParts(binary: { id: number; size: number | null }) { if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete() }