implemented a lot of features

commands/workers/File.ts

@@ -1,13 +1,14 @@
 import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
+import db from '@adonisjs/lucid/services/db'
 import queueConfig from '#config/queue'
+import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
-import File from '#models/file'
 
 export default class FileWorker extends BaseCommand {
   public static commandName = 'worker:file'
-  public static description = 'Starts a worker to process completed files.'
+  public static description = 'Starts a worker that turns completed binaries into files.'
 
   public static options: CommandOptions = {
     startApp: true,
@@ -19,45 +20,78 @@ export default class FileWorker extends BaseCommand {
     const collectionQueue = QueueService.collectionQueue
 
     const worker = new Worker('file-queue', async (job) => {
-      const { filename, parts, groups } = job.data
-      const partCount = Object.keys(parts).length
-      this.logger.debug(`Processing complete file: "${filename}" with ${partCount} parts.`)
+      const { binaryId } = job.data
 
-      const firstPart = JSON.parse(Object.values(parts)[0] as string)
-      const poster = firstPart.from
-      const date = new Date(firstPart.date).getTime()
-
-      const messageIds = Object.entries(parts).reduce((acc, [partNumber, partData]) => {
-        const part = JSON.parse(partData as string)
-        const messageId = part['message-id']
-        if (messageId) {
-          acc[partNumber] = {
-            id: messageId.replace(/[<>]/g, ''),
-            size: part[':bytes'],
-          }
-        } else {
-          this.logger.warning(`Message ID not found for part ${partNumber} of file "${filename}"`)
-        }
-        return acc
-      }, {} as Record<string, any>)
-
-      if (Object.keys(messageIds).length !== partCount) {
-        throw new Error(`Could not process all parts for file "${filename}" due to missing message IDs.`)
+      const binary = await db.from('binaries').where('id', binaryId).first()
+      if (!binary) {
+        this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`)
+        return
       }
 
-      const file = await File.create({
-        filename,
-        poster,
-        date,
-        parts: partCount,
-        messageIds,
-        groups,
-      })
+      if (binary.conflicted_at) {
+        this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`)
+        return
+      }
 
-      this.logger.debug(`Saved file "${filename}" to database with ID: ${file.id}`)
+      const parts = await db
+        .from('parts')
+        .where('binary_id', binaryId)
+        .orderBy('number')
+        .select('number', 'message_id', 'bytes')
+      if (parts.length < binary.total_parts) {
+        throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
+      }
 
-      await collectionQueue.add('process-collection', { fileId: file.id })
-      this.logger.debug(`Added file ID ${file.id} to collection queue.`)
+      const messageIds = Object.fromEntries(
+        parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }])
+      )
+
+      // Already known as a file, e.g. added from a posted NZB before its own articles completed.
+      const existing = await db
+        .from('files')
+        .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? ''])
+        .first()
+      if (existing) {
+        await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
+        await dropCopiedParts(binary)
+        this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`)
+        return
+      }
+
+      // file_key is unique: a second completion of the same upload inserts nothing.
+      const knex = db.connection().getWriteClient()
+      const now = new Date()
+      const [inserted] = await knex('files')
+        .insert({
+          file_key: binary.file_key,
+          filename: binary.name,
+          poster: binary.poster,
+          date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
+          parts: parts.length,
+          message_ids: JSON.stringify(messageIds),
+          groups: JSON.stringify(binary.groups),
+          release_id: binary.release_id,
+          real_name: binary.real_name,
+          size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
+          created_at: now,
+          updated_at: now,
+        })
+        .onConflict('file_key')
+        .ignore()
+        .returning('id')
+
+      const fileId =
+        inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
+      await db.from('binaries').where('id', binaryId).update({ file_id: fileId })
+      await dropCopiedParts(binary)
+
+      if (!inserted) {
+        this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`)
+        return
+      }
+
+      this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
+      await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
     }, { connection: queueConfig.connection })
 
     worker.on('failed', (job, err) => {
@@ -65,6 +99,19 @@ export default class FileWorker extends BaseCommand {
     })
 
     this.logger.info('File worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await QueueService.closeAll()
   }
 }
+
+/**
+ * The file holds every message id now. Binaries identified by their subject
+ * (no size: header-derived) don't need their parts any more; body-derived
+ * ones keep them, so articles of an equal-sized next volume are recognised
+ * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
+ */
+async function dropCopiedParts(binary: { id: number; size: number | null }) {
+  if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete()
+}