118 lines
4.4 KiB
TypeScript
118 lines
4.4 KiB
TypeScript
import { BaseCommand } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import { Worker } from 'bullmq'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import queueConfig from '#config/queue'
|
|
import Shutdown from '#services/Shutdown'
|
|
import QueueService from '#services/QueueService'
|
|
|
|
export default class FileWorker extends BaseCommand {
|
|
public static commandName = 'worker:file'
|
|
public static description = 'Starts a worker that turns completed binaries into files.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
public async run() {
|
|
this.logger.info('Starting file worker...')
|
|
|
|
const collectionQueue = QueueService.collectionQueue
|
|
|
|
const worker = new Worker('file-queue', async (job) => {
|
|
const { binaryId } = job.data
|
|
|
|
const binary = await db.from('binaries').where('id', binaryId).first()
|
|
if (!binary) {
|
|
this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`)
|
|
return
|
|
}
|
|
|
|
if (binary.conflicted_at) {
|
|
this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`)
|
|
return
|
|
}
|
|
|
|
const parts = await db
|
|
.from('parts')
|
|
.where('binary_id', binaryId)
|
|
.orderBy('number')
|
|
.select('number', 'message_id', 'bytes')
|
|
if (parts.length < binary.total_parts) {
|
|
throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
|
|
}
|
|
|
|
const messageIds = Object.fromEntries(
|
|
parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }])
|
|
)
|
|
|
|
// Already known as a file, e.g. added from a posted NZB before its own articles completed.
|
|
const existing = await db
|
|
.from('files')
|
|
.whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? ''])
|
|
.first()
|
|
if (existing) {
|
|
await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
|
|
await dropCopiedParts(binary)
|
|
this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`)
|
|
return
|
|
}
|
|
|
|
// file_key is unique: a second completion of the same upload inserts nothing.
|
|
const knex = db.connection().getWriteClient()
|
|
const now = new Date()
|
|
const [inserted] = await knex('files')
|
|
.insert({
|
|
file_key: binary.file_key,
|
|
filename: binary.name,
|
|
poster: binary.poster,
|
|
date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
|
|
parts: parts.length,
|
|
message_ids: JSON.stringify(messageIds),
|
|
groups: JSON.stringify(binary.groups),
|
|
release_id: binary.release_id,
|
|
real_name: binary.real_name,
|
|
size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
|
|
created_at: now,
|
|
updated_at: now,
|
|
})
|
|
.onConflict('file_key')
|
|
.ignore()
|
|
.returning('id')
|
|
|
|
const fileId =
|
|
inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
|
|
await db.from('binaries').where('id', binaryId).update({ file_id: fileId })
|
|
await dropCopiedParts(binary)
|
|
|
|
if (!inserted) {
|
|
this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`)
|
|
return
|
|
}
|
|
|
|
this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
|
|
await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
|
|
}, { connection: queueConfig.connection })
|
|
|
|
worker.on('failed', (job, err) => {
|
|
this.logger.error(`File job ${job?.id} failed: ${err.message}`)
|
|
})
|
|
|
|
this.logger.info('File worker started and listening for jobs.')
|
|
await Shutdown.wait()
|
|
this.logger.info('Shutting down: finishing active jobs...')
|
|
await worker.close()
|
|
await QueueService.closeAll()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The file holds every message id now. Binaries identified by their subject
|
|
* (no size: header-derived) don't need their parts any more; body-derived
|
|
* ones keep them, so articles of an equal-sized next volume are recognised
|
|
* as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
|
|
*/
|
|
async function dropCopiedParts(binary: { id: number; size: number | null }) {
|
|
if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete()
|
|
}
|