implemented a lot of features

commands/workers/Body.ts

@@ -4,9 +4,13 @@ import { Worker } from 'bullmq'
 import fs from 'node:fs/promises'
 import path from 'node:path'
 import queueConfig from '#config/queue'
+import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
+import env from '#start/env'
 import NntpService from '#services/NntpService'
-import { parseYencMeta } from '#services/YencService'
+import IngestService, { segmentFromYenc } from '#services/IngestService'
+import { parseYencMeta, YencError } from '#lib/yenc/index'
+import { firstChunkMd5 } from '#services/ContentService'
 
 export default class BodyWorker extends BaseCommand {
   public static commandName = 'worker:body'
@@ -20,31 +24,26 @@ export default class BodyWorker extends BaseCommand {
     this.logger.info('Starting body worker...')
 
     const pool = NntpService
-    const headerQueue = QueueService.headerQueue
 
     const worker = new Worker('body-queue', async (job) => {
-      const { header, group } = job.data
+      const { header, group, articleNumber } = job.data
       const messageId = header['message-id']
       this.logger.debug(`Processing header with unparsable subject: ${header.subject}`)
 
       let conn
+      let failed = false
       try {
         conn = await pool.acquire()
-        const bodyBuffer: Buffer = (await conn.body(messageId)).data
+        const bodyBuffer = await pool.fetchBody(conn, messageId)
 
+        let meta
         try {
-          const meta = parseYencMeta(bodyBuffer)
-          if (meta.header.name) {
-            const { name, part, total } = meta.header
-            const newSubject = `"${name}" yEnc (${part}/${total})`
-            header.subject = newSubject
-
-            this.logger.info(`Found yEnc metadata in body. New subject: ${newSubject}`)
-            await headerQueue.add('process-header', { header, group })
-          } else {
-            this.logger.warning(`Could not find yEnc metadata in body for header: ${header.subject}`)
-          }
+          meta = parseYencMeta(bodyBuffer)
         } catch (parseError: any) {
+          if (parseError instanceof YencError && bodyBuffer.indexOf('=ybegin ') === -1) {
+            this.logger.debug(`Not a yEnc post, skipping: ${messageId}`)
+            return
+          }
           this.logger.error(`Failed to parse yEnc data for message ID ${messageId}. Dumping buffer.`)
           const debugDir = path.join(this.app.appRoot.pathname, 'debug')
           await fs.mkdir(debugDir, { recursive: true })
@@ -54,21 +53,45 @@ export default class BodyWorker extends BaseCommand {
           this.logger.error(`Problematic body buffer saved to: ${dumpFile}`)
           throw parseError
         }
+
+        pool.release(conn)
+        conn = undefined
+
+        const segment = segmentFromYenc(header, group, meta, articleNumber)
+        if (segment?.number === 1) segment.md5_16k = firstChunkMd5(bodyBuffer, meta.begin.size)
+        if (!segment) {
+          this.logger.warning(`Inconsistent yEnc part numbering in ${messageId}, skipping.`)
+          return
+        }
+        const completed = await IngestService.ingest([segment])
+        this.logger.debug(
+          `Stored part ${segment.number}/${segment.totalParts} of "${segment.name}"` +
+            (completed.length > 0 ? ' (file complete)' : '')
+        )
       } catch (error: any) {
+        failed = true
         this.logger.error(`Error in body worker for message ID ${messageId}: ${error.message}`)
         throw error
       } finally {
         if (conn) {
-          pool.release(conn)
+          pool.release(conn, failed)
         }
       }
-    }, { connection: queueConfig.connection })
+    }, {
+      connection: queueConfig.connection,
+      // Each job holds one NNTP connection while downloading, so keep this <= the pool size.
+      concurrency: env.get('BODY_CONCURRENCY', 1),
+    })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`Body job ${job?.id} failed: ${err.message}`)
     })
 
     this.logger.info('Body worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
   }
 }

commands/workers/Collection.ts

@@ -2,10 +2,12 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import queueConfig from '#config/queue'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import QueueService from '#services/QueueService'
 import NntpService from '#services/NntpService'
-import { YencFile } from '#services/YencFile'
+import ContentService from '#services/ContentService'
 import File from '#models/file'
-import { createExtractorFromData } from 'node-unrar-js'
 
 export default class CollectionWorker extends BaseCommand {
   public static commandName = 'worker:collection'
@@ -18,8 +20,6 @@ export default class CollectionWorker extends BaseCommand {
   public async run() {
     this.logger.info('Starting collection worker...')
 
-    const pool = NntpService
-
     const worker = new Worker('collection-queue', async (job) => {
       const { fileId } = job.data
       this.logger.debug(`Processing file ID ${fileId} for collection.`)
@@ -30,51 +30,47 @@ export default class CollectionWorker extends BaseCommand {
         return
       }
 
-      const RAR_REGEX = /\.part0*1\.rar$/
-      if (RAR_REGEX.test(file.filename)) {
-        this.logger.info(`File "${file.filename}" is the first part of a RAR set.`)
-
-        const firstPart = file.messageIds['1']
-        if (!firstPart || !firstPart.id) {
-          this.logger.error(`Could not find message ID for the first part of file "${file.filename}".`)
-          return
+      try {
+        const content = await ContentService.inspect(fileId)
+        if (content?.par2) {
+          const { par2 } = content
+          this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
+        } else if (content?.archive) {
+          const { info, releaseRenamed, grouped } = content.archive
+          const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
+            .filter(Boolean)
+            .join(', ')
+          this.logger.info(
+            `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
+              (grouped ? ', grouped into a release' : '') +
+              (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
+          )
         }
-
-        let conn
-        try {
-          conn = await pool.acquire()
-          const bodyBuffer = (await conn.body(`<${firstPart.id}>`)).data
-
-          const yencFile = new YencFile()
-          yencFile.processPart(bodyBuffer)
-          const decodedBuffer = yencFile.getBuffer()
-
-          const extractor = await createExtractorFromData({ data: new Uint8Array(decodedBuffer).buffer })
-          const fileList = extractor.getFileList()
-          // In a real implementation, we would save this file list.
-          this.logger.info(`Files in "${file.filename}": ${JSON.stringify(fileList)}`)
-
-        } catch (error: any) {
-          if (error.code === 430) {
-            this.logger.error(`Article not found for first part of RAR set (Message ID: ${firstPart.id})`)
-          } else {
-            this.logger.error(`Error processing RAR file: ${error.message}`)
-          }
-        } finally {
-          if (conn) {
-            pool.release(conn)
-          }
+        if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
+        if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
+        if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
+        if (content?.nzb) {
+          const n = content.nzb
+          this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
         }
-      } else {
-        this.logger.debug(`File "${file.filename}" is not the first part of a RAR set.`)
+      } catch (error: any) {
+        this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
       }
-    }, { connection: queueConfig.connection })
+    }, {
+      connection: queueConfig.connection,
+      // Each job holds one NNTP connection while downloading; keep <= the pool size.
+      concurrency: env.get('COLLECTION_CONCURRENCY', 1),
+    })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
     })
 
     this.logger.info('Collection worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
   }
 }

commands/workers/Deferred.ts

@@ -0,0 +1,57 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import DeferredService from '#services/DeferredService'
+
+const IDLE_MS = 60_000
+
+/**
+ * Works through `deferred_files` (CONCEPT.md D28): one body per file, and
+ * small files completely. Not started by default (compose profile
+ * `deferred`); run it when there is bandwidth to spare.
+ */
+export default class DeferredWorker extends BaseCommand {
+  public static commandName = 'worker:deferred'
+  public static description = 'Samples deferred files (e.g. ngPost uploads): one article body per file.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Stop after this many files (default: keep running)' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = env.get('DEFERRED_CONCURRENCY', 4)
+    this.logger.info(`Deferred worker started (concurrency ${concurrency}).`)
+    const counts: Record<string, number> = {}
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        if (batch.length === 0) {
+          if (this.limit) break
+          await Shutdown.sleep(IDLE_MS)
+          continue
+        }
+        let next = 0
+        await Promise.all(
+          Array.from({ length: concurrency }, async () => {
+            while (next < batch.length) {
+              const status = await DeferredService.process(batch[next++])
+              counts[status] = (counts[status] ?? 0) + 1
+            }
+          })
+        )
+        done += batch.length
+        this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+      }
+    } finally {
+      await NntpService.shutdown()
+      await QueueService.closeAll()
+    }
+  }
+}

commands/workers/Fetch.ts

@@ -2,12 +2,20 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import queueConfig from '#config/queue'
+import env from '#start/env'
+import db from '@adonisjs/lucid/services/db'
+import indexerConfig from '#config/indexer'
+import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
 import NntpService from '#services/NntpService'
+import FetchRangeService from '#services/FetchRangeService'
+import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
+import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService'
+import SpotService, { isSpotGroup } from '#services/SpotService'
 
 export default class FetchWorker extends BaseCommand {
   public static commandName = 'worker:fetch'
-  public static description = 'Starts a worker to fetch headers from the NNTP server.'
+  public static description = 'Starts a worker that fetches headers (XOVER) and stores them as binaries/parts.'
 
   public static options: CommandOptions = {
     startApp: true,
@@ -17,53 +25,86 @@ export default class FetchWorker extends BaseCommand {
     this.logger.info('Starting fetch worker...')
 
     const pool = NntpService
-    const headerQueue = QueueService.headerQueue
+    const bodyQueue = QueueService.bodyQueue
 
     const worker = new Worker('nntp-fetch-queue', async (job) => {
       const { groupName, startId, endId } = job.data
       this.logger.info(`Processing fetch job for ${groupName}, articles ${startId}-${endId}`)
 
+      let overviews: [number, Record<string, string>][]
       let conn
+      let failed = false
       try {
         conn = await pool.acquire()
         await conn.group(groupName)
-        const overview: any = await conn.xover(startId, endId)
-        this.logger.info(`Fetched ${overview.overviews.length} headers from ${groupName}.`)
-
-        if (overview.overviews.length > 0) {
-          const jobs = overview.overviews.map(([id, header]: [number, any]) => {
-            if (!header) {
-              this.logger.warning(`Header is undefined for job ${job.id}`)
-              return null
-            }
-            return {
-              name: 'process-header',
-              data: { header, group: groupName },
-              opts: { jobId: `${groupName}-${id}` },
-            }
-          }).filter(Boolean)
-
-          await headerQueue.addBulk(jobs)
-          this.logger.info(`Added ${jobs.length} header jobs to the queue for ${groupName}.`)
-        }
+        overviews = (await conn.xover(startId, endId)).overviews
       } catch (error: any) {
+        failed = true
         this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`)
         throw error
       } finally {
         if (conn) {
-          pool.release(conn)
+          pool.release(conn, failed)
         }
       }
-    }, { 
+
+      if (isSpotGroup(groupName)) {
+        const spots = await SpotService.recordHeaders(groupName, overviews)
+        if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
+        this.logger.info(`${groupName} ${startId}-${endId}: ${overviews.length} headers, ${spots} spots queued.`)
+        return
+      }
+
+      const group = await db.from('groups').where('name', groupName).first()
+      const fetchBodies = Boolean(group?.fetch_bodies)
+
+      const segments: IngestSegment[] = []
+      const obfuscated: { articleNumber: number; header: Record<string, string> }[] = []
+      const deferred: DeferredArticle[] = []
+      let skipped = 0
+      for (const [articleNumber, header] of overviews) {
+        if (!header) continue
+        const segment = segmentFromHeader(header, groupName, articleNumber)
+        if (segment) segments.push(segment)
+        // Recorded per file (or as article ranges) in every group, bodies or not; processed later if at all.
+        else if (isDeferred(header)) deferred.push({ articleNumber, header })
+        else if (!fetchBodies || indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
+        else obfuscated.push({ articleNumber, header })
+      }
+
+      const completed = await IngestService.ingest(segments)
+      const deferredFiles = await DeferredService.record(groupName, deferred)
+
+      if (obfuscated.length > 0) {
+        await bodyQueue.addBulk(obfuscated.map(({ articleNumber, header }) => bodyJob(header, groupName, articleNumber)))
+      }
+
+      if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
+
+      this.logger.info(
+        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${segments.length} stored, ` +
+          `${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` +
+          `${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.`
+      )
+    }, {
       connection: queueConfig.connection,
-      concurrency: 5,
+      // XOVER is quick; most of a job is storing. Each concurrent job uses one DB connection (DB_POOL_MAX).
+      concurrency: env.get('FETCH_CONCURRENCY', 5),
     })
 
-    worker.on('failed', (job, err) => {
+    worker.on('failed', async (job, err) => {
       this.logger.error(`Fetch job ${job?.id} failed for group ${job?.data.groupName}: ${err.message}`)
+      // Only after the last attempt; the scheduler re-queues the range later.
+      if (job?.data.rangeId && job.attemptsMade >= (job.opts.attempts ?? 1)) {
+        await FetchRangeService.markFailed(job.data.rangeId, err.message).catch(() => {})
+      }
     })
 
     this.logger.info('Fetch worker started and listening for jobs.')
-    await new Promise(() => {}) // Keep command running
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
   }
 }

commands/workers/File.ts

@@ -1,13 +1,14 @@
 import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
+import db from '@adonisjs/lucid/services/db'
 import queueConfig from '#config/queue'
+import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
-import File from '#models/file'
 
 export default class FileWorker extends BaseCommand {
   public static commandName = 'worker:file'
-  public static description = 'Starts a worker to process completed files.'
+  public static description = 'Starts a worker that turns completed binaries into files.'
 
   public static options: CommandOptions = {
     startApp: true,
@@ -19,45 +20,78 @@ export default class FileWorker extends BaseCommand {
     const collectionQueue = QueueService.collectionQueue
 
     const worker = new Worker('file-queue', async (job) => {
-      const { filename, parts, groups } = job.data
-      const partCount = Object.keys(parts).length
-      this.logger.debug(`Processing complete file: "${filename}" with ${partCount} parts.`)
+      const { binaryId } = job.data
 
-      const firstPart = JSON.parse(Object.values(parts)[0] as string)
-      const poster = firstPart.from
-      const date = new Date(firstPart.date).getTime()
-
-      const messageIds = Object.entries(parts).reduce((acc, [partNumber, partData]) => {
-        const part = JSON.parse(partData as string)
-        const messageId = part['message-id']
-        if (messageId) {
-          acc[partNumber] = {
-            id: messageId.replace(/[<>]/g, ''),
-            size: part[':bytes'],
-          }
-        } else {
-          this.logger.warning(`Message ID not found for part ${partNumber} of file "${filename}"`)
-        }
-        return acc
-      }, {} as Record<string, any>)
-
-      if (Object.keys(messageIds).length !== partCount) {
-        throw new Error(`Could not process all parts for file "${filename}" due to missing message IDs.`)
+      const binary = await db.from('binaries').where('id', binaryId).first()
+      if (!binary) {
+        this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`)
+        return
       }
 
-      const file = await File.create({
-        filename,
-        poster,
-        date,
-        parts: partCount,
-        messageIds,
-        groups,
-      })
+      if (binary.conflicted_at) {
+        this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`)
+        return
+      }
 
-      this.logger.debug(`Saved file "${filename}" to database with ID: ${file.id}`)
+      const parts = await db
+        .from('parts')
+        .where('binary_id', binaryId)
+        .orderBy('number')
+        .select('number', 'message_id', 'bytes')
+      if (parts.length < binary.total_parts) {
+        throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
+      }
 
-      await collectionQueue.add('process-collection', { fileId: file.id })
-      this.logger.debug(`Added file ID ${file.id} to collection queue.`)
+      const messageIds = Object.fromEntries(
+        parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }])
+      )
+
+      // Already known as a file, e.g. added from a posted NZB before its own articles completed.
+      const existing = await db
+        .from('files')
+        .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? ''])
+        .first()
+      if (existing) {
+        await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
+        await dropCopiedParts(binary)
+        this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`)
+        return
+      }
+
+      // file_key is unique: a second completion of the same upload inserts nothing.
+      const knex = db.connection().getWriteClient()
+      const now = new Date()
+      const [inserted] = await knex('files')
+        .insert({
+          file_key: binary.file_key,
+          filename: binary.name,
+          poster: binary.poster,
+          date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
+          parts: parts.length,
+          message_ids: JSON.stringify(messageIds),
+          groups: JSON.stringify(binary.groups),
+          release_id: binary.release_id,
+          real_name: binary.real_name,
+          size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
+          created_at: now,
+          updated_at: now,
+        })
+        .onConflict('file_key')
+        .ignore()
+        .returning('id')
+
+      const fileId =
+        inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
+      await db.from('binaries').where('id', binaryId).update({ file_id: fileId })
+      await dropCopiedParts(binary)
+
+      if (!inserted) {
+        this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`)
+        return
+      }
+
+      this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
+      await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
     }, { connection: queueConfig.connection })
 
     worker.on('failed', (job, err) => {
@@ -65,6 +99,19 @@ export default class FileWorker extends BaseCommand {
     })
 
     this.logger.info('File worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await QueueService.closeAll()
   }
 }
+
+/**
+ * The file holds every message id now. Binaries identified by their subject
+ * (no size: header-derived) don't need their parts any more; body-derived
+ * ones keep them, so articles of an equal-sized next volume are recognised
+ * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
+ */
+async function dropCopiedParts(binary: { id: number; size: number | null }) {
+  if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete()
+}

commands/workers/Header.ts

@@ -1,65 +0,0 @@
-import { BaseCommand } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import { Worker } from 'bullmq'
-import queueConfig from '#config/queue'
-import QueueService from '#services/QueueService'
-import RedisService from '#services/RedisService'
-
-export default class HeaderWorker extends BaseCommand {
-  public static commandName = 'worker:header'
-  public static description = 'Starts a worker to process headers.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  public async run() {
-    this.logger.info('Starting header worker...')
-
-    const redis = RedisService.client
-    const fileQueue = QueueService.fileQueue
-    const bodyQueue = QueueService.bodyQueue
-
-    const worker = new Worker('header-queue', async (job) => {
-      const { header, group } = job.data
-      if (!header || !header.subject) {
-        this.logger.warning(`Received job with invalid header data. JobID: ${job.id}`)
-        return
-      }
-
-      const subject = header.subject
-      const SUBJECT_REGEX = /"(.+)"(?: yEnc)? \((\d+)\/(\d+)\)/
-      const match = subject.match(SUBJECT_REGEX)
-
-      if (match) {
-        const filename = match[1]
-        const part = parseInt(match[2], 10)
-        const total = parseInt(match[3], 10)
-
-        const fileKey = `file:${filename}`
-        await redis.hset(fileKey, part, JSON.stringify(header))
-
-        const partCount = await redis.hlen(fileKey)
-
-        if (partCount === total) {
-          const fileParts = await redis.hgetall(fileKey)
-          await fileQueue.add('process-file', { filename, parts: fileParts, groups: [group] })
-          await redis.del(fileKey)
-          this.logger.info(`File "${filename}" is complete and moved to file-queue.`)
-        } else {
-          this.logger.info(`Stored part ${part}/${total} for file "${filename}"`)
-        }
-      } else {
-        this.logger.warning(`Could not parse subject: "${subject}". Moving to body-queue.`)
-        await bodyQueue.add('process-body', { header, group })
-      }
-    }, { connection: queueConfig.connection })
-
-    worker.on('failed', (job, err) => {
-      this.logger.error(`Header job ${job?.id} failed: ${err.message}`)
-    })
-    
-    this.logger.info('Header worker started and listening for jobs.')
-    await new Promise(() => {})
-  }
-}

commands/workers/Spots.ts

@@ -0,0 +1,54 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+const IDLE_MS = 30_000
+
+/**
+ * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
+ * downloads its compressed NZB (a few KB to a few MB) and imports it.
+ */
+export default class SpotsWorker extends BaseCommand {
+  public static commandName = 'worker:spots'
+  public static description = 'Reads Spotnet spots: verifies them and imports their NZBs.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Stop after this many spots (default: keep running)' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = env.get('SPOT_CONCURRENCY', 4)
+    this.logger.info(`Spot worker started (concurrency ${concurrency}).`)
+    const counts: Record<string, number> = {}
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const batch = await SpotService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        if (batch.length === 0) {
+          if (this.limit) break
+          await Shutdown.sleep(IDLE_MS)
+          continue
+        }
+        let next = 0
+        await Promise.all(
+          Array.from({ length: concurrency }, async () => {
+            while (next < batch.length) {
+              const status = await SpotService.process(batch[next++])
+              counts[status] = (counts[status] ?? 0) + 1
+            }
+          })
+        )
+        done += batch.length
+        this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}