implemented a lot of features

commands/workers/Collection.ts

@@ -2,10 +2,12 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import queueConfig from '#config/queue'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import QueueService from '#services/QueueService'
 import NntpService from '#services/NntpService'
-import { YencFile } from '#services/YencFile'
+import ContentService from '#services/ContentService'
 import File from '#models/file'
-import { createExtractorFromData } from 'node-unrar-js'
 
 export default class CollectionWorker extends BaseCommand {
   public static commandName = 'worker:collection'
@@ -18,8 +20,6 @@ export default class CollectionWorker extends BaseCommand {
   public async run() {
     this.logger.info('Starting collection worker...')
 
-    const pool = NntpService
-
     const worker = new Worker('collection-queue', async (job) => {
       const { fileId } = job.data
       this.logger.debug(`Processing file ID ${fileId} for collection.`)
@@ -30,51 +30,47 @@ export default class CollectionWorker extends BaseCommand {
         return
       }
 
-      const RAR_REGEX = /\.part0*1\.rar$/
-      if (RAR_REGEX.test(file.filename)) {
-        this.logger.info(`File "${file.filename}" is the first part of a RAR set.`)
-
-        const firstPart = file.messageIds['1']
-        if (!firstPart || !firstPart.id) {
-          this.logger.error(`Could not find message ID for the first part of file "${file.filename}".`)
-          return
+      try {
+        const content = await ContentService.inspect(fileId)
+        if (content?.par2) {
+          const { par2 } = content
+          this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
+        } else if (content?.archive) {
+          const { info, releaseRenamed, grouped } = content.archive
+          const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
+            .filter(Boolean)
+            .join(', ')
+          this.logger.info(
+            `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
+              (grouped ? ', grouped into a release' : '') +
+              (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
+          )
         }
-
-        let conn
-        try {
-          conn = await pool.acquire()
-          const bodyBuffer = (await conn.body(`<${firstPart.id}>`)).data
-
-          const yencFile = new YencFile()
-          yencFile.processPart(bodyBuffer)
-          const decodedBuffer = yencFile.getBuffer()
-
-          const extractor = await createExtractorFromData({ data: new Uint8Array(decodedBuffer).buffer })
-          const fileList = extractor.getFileList()
-          // In a real implementation, we would save this file list.
-          this.logger.info(`Files in "${file.filename}": ${JSON.stringify(fileList)}`)
-
-        } catch (error: any) {
-          if (error.code === 430) {
-            this.logger.error(`Article not found for first part of RAR set (Message ID: ${firstPart.id})`)
-          } else {
-            this.logger.error(`Error processing RAR file: ${error.message}`)
-          }
-        } finally {
-          if (conn) {
-            pool.release(conn)
-          }
+        if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
+        if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
+        if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
+        if (content?.nzb) {
+          const n = content.nzb
+          this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
         }
-      } else {
-        this.logger.debug(`File "${file.filename}" is not the first part of a RAR set.`)
+      } catch (error: any) {
+        this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
       }
-    }, { connection: queueConfig.connection })
+    }, {
+      connection: queueConfig.connection,
+      // Each job holds one NNTP connection while downloading; keep <= the pool size.
+      concurrency: env.get('COLLECTION_CONCURRENCY', 1),
+    })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
     })
 
     this.logger.info('Collection worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
   }
 }