implemented a lot of features

commands/BodyQueuePrune.ts

@@ -0,0 +1,97 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import indexerConfig from '#config/indexer'
+import QueueService from '#services/QueueService'
+import RedisService from '#services/RedisService'
+import DeferredService, { isDeferred } from '#services/DeferredService'
+
+const BATCH = 2000
+
+/** Rebuilds the wait list (KEYS[1]) without the ids in the set KEYS[2], in one atomic pass. */
+const FILTER_LIST = `
+local ids = redis.call('LRANGE', KEYS[1], 0, -1)
+local keep = {}
+for _, id in ipairs(ids) do
+  if redis.call('SISMEMBER', KEYS[2], id) == 0 then keep[#keep + 1] = id end
+end
+redis.call('DEL', KEYS[1])
+for i = 1, #keep, 5000 do
+  redis.call('RPUSH', KEYS[1], unpack(keep, i, math.min(i + 4999, #keep)))
+end
+return #ids - #keep`
+
+/**
+ * Removes waiting body jobs whose message-id is on the skip or defer list,
+ * e.g. after adding a pattern (CONCEPT.md D24, D28). Deferred ones are
+ * recorded in deferred_files first, so they can still be processed later.
+ */
+export default class BodyQueuePrune extends BaseCommand {
+  public static commandName = 'body-queue:prune'
+  public static description = 'Drops waiting body jobs that are now skipped or deferred.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count' })
+  declare dryRun: boolean
+
+  @flags.boolean({ description: 'Drop deferred jobs without recording them (already recorded by an earlier run)' })
+  declare skipRecord: boolean
+
+  public async run() {
+    const redis = RedisService.client
+    const queue = QueueService.bodyQueue
+    const waitKey = queue.toKey('wait')
+    let skipped = 0
+    let deferred = 0
+    let scanned = 0
+    const dropped: string[] = []
+    try {
+      const ids: string[] = await redis.lrange(waitKey, 0, -1)
+      for (let i = 0; i < ids.length; i += BATCH) {
+        const batch = ids.slice(i, i + BATCH)
+        const data: [Error | null, string | null][] = await batch
+          .reduce((pipeline, id) => pipeline.hget(queue.toKey(id), 'data'), redis.pipeline())
+          .exec()
+        const drop: string[] = []
+        const toDefer = new Map<string, { articleNumber: number; header: Record<string, string> }[]>()
+        batch.forEach((id, j) => {
+          const job = data[j][1] ? JSON.parse(data[j][1]!) : null
+          if (!job?.header) return
+          const messageId = job.header['message-id'] ?? ''
+          if (isDeferred(job.header)) {
+            const list = toDefer.get(job.group) ?? []
+            list.push({ articleNumber: Number(job.articleNumber), header: job.header })
+            toDefer.set(job.group, list)
+            drop.push(id)
+            deferred++
+          } else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(messageId))) {
+            drop.push(id)
+            skipped++
+          }
+        })
+        scanned += batch.length
+        if (this.dryRun || drop.length === 0) continue
+        if (!this.skipRecord) for (const [group, articles] of toDefer) await DeferredService.record(group, articles)
+        dropped.push(...drop)
+      }
+      if (dropped.length > 0) {
+        const dropKey = queue.toKey('prune-ids')
+        await redis.del(dropKey)
+        for (let i = 0; i < dropped.length; i += BATCH) await redis.sadd(dropKey, ...dropped.slice(i, i + BATCH))
+        await redis.eval(FILTER_LIST, 2, waitKey, dropKey)
+        await redis.del(dropKey)
+        for (let i = 0; i < dropped.length; i += BATCH) {
+          await redis.unlink(...dropped.slice(i, i + BATCH).map((id) => queue.toKey(id)))
+        }
+      }
+      this.logger.info(
+        `Scanned ${scanned} waiting body jobs: ${this.dryRun ? 'would drop' : 'dropped'} ${deferred} deferred and ${skipped} skipped.`
+      )
+    } finally {
+      await QueueService.closeAll()
+      await RedisService.close()
+    }
+  }
+}

commands/ContentBackfill.ts

@@ -0,0 +1,68 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import QueueService from '#services/QueueService'
+import ContentService from '#services/ContentService'
+
+const BATCH = 1000
+
+/**
+ * Queues completed files for content inspection (names and groupings from
+ * par2 and RAR headers, CONCEPT.md D21) that were completed before it
+ * existed. Costs one article download per file, done by worker:collection.
+ */
+export default class ContentBackfill extends BaseCommand {
+  public static commandName = 'content:backfill'
+  public static description = 'Queues existing files with random names for content inspection.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Queue at most this many files' })
+  declare limit?: number
+
+  @flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' })
+  declare all: boolean
+
+  @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
+  declare archiveNzbs: boolean
+
+  public async run() {
+    let lastId = 0
+    let queued = 0
+    try {
+      while (!this.limit || queued < this.limit) {
+        // Files never inspected: no fingerprint on their binary and no archive listing.
+        const query = db
+          .from('files as f')
+          .join('binaries as b', 'b.file_id', 'f.id')
+          .where('f.id', '>', lastId)
+        if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
+        else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
+        const files = await query
+          .orderBy('f.id')
+          .limit(BATCH)
+          .select('f.id', 'f.filename')
+        if (files.length === 0) break
+        lastId = Number(files.at(-1).id)
+
+        const jobs = files
+          .filter((f) => ContentService.isCandidate(f.filename))
+          .slice(0, this.limit ? this.limit - queued : undefined)
+          .map((f) => ({
+            name: 'process-collection',
+            data: { fileId: Number(f.id) },
+            // A new id per run: BullMQ ignores an add() for an id it still knows.
+            // Targeted backfills go ahead of whatever is waiting.
+            opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
+          }))
+        if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+        queued += jobs.length
+      }
+      this.logger.info(`Queued ${queued} files for content inspection.`)
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+}

commands/DeferredExpandRanges.ts

@@ -0,0 +1,32 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import DeferredService from '#services/DeferredService'
+
+/**
+ * Sends recorded deferred ranges (fully obfuscated ngPost articles, CONCEPT.md
+ * D28) to the body queue, oldest first. Every article in them is downloaded:
+ * their headers can't be grouped into files.
+ */
+export default class DeferredExpandRanges extends BaseCommand {
+  public static commandName = 'deferred:expand-ranges'
+  public static description = 'Queues the bodies of deferred article ranges (costly: every article is downloaded).'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Ranges to expand', default: 10 })
+  declare limit: number
+
+  public async run() {
+    try {
+      const { ranges, articles } = await DeferredService.expandRanges(this.limit)
+      this.logger.info(`Expanded ${ranges} ranges: ${articles} articles sent to the body queue.`)
+    } finally {
+      await NntpService.shutdown()
+      await QueueService.closeAll()
+    }
+  }
+}

commands/GcScheduler.ts

@@ -0,0 +1,282 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import type { Queue } from 'bullmq'
+import db from '@adonisjs/lucid/services/db'
+import gcConfig from '#config/gc'
+import QueueService from '#services/QueueService'
+import RedisService from '#services/RedisService'
+import Shutdown from '#services/Shutdown'
+
+const CLEAN_BATCH = 1000
+const SCAN_COUNT = 1000
+
+export default class GcScheduler extends BaseCommand {
+  public static commandName = 'gc:scheduler'
+  public static description = 'Periodically removes old BullMQ jobs and stale binaries.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Run one sweep and exit' })
+  declare once: boolean
+
+  @flags.boolean({ description: 'Report what would be removed without changing anything' })
+  declare dryRun: boolean
+
+  public async run() {
+    this.logger.info(
+      `GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
+    )
+
+    try {
+      do {
+        try {
+          await this.sweep()
+        } catch (err: any) {
+          this.logger.error(`GC sweep failed: ${err.message}`)
+        }
+        if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
+      } while (!this.once && !Shutdown.requested)
+    } finally {
+      await QueueService.closeAll()
+      await RedisService.close()
+    }
+  }
+
+  private async sweep() {
+    const { completedAgeSeconds, failedAgeSeconds } = gcConfig.jobs
+    for (const queue of [...QueueService.all, ...QueueService.legacy]) {
+      const completed = await this.cleanJobs(queue, 'completed', completedAgeSeconds)
+      const failed = await this.cleanJobs(queue, 'failed', failedAgeSeconds)
+      if (completed || failed) {
+        this.logger.info(
+          `${queue.name}: ${this.verb('remove', 'removed')} ${completed} completed and ${failed} failed jobs.`
+        )
+      }
+    }
+
+    const trimmed = await this.trimBodyQueue()
+    if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`)
+
+    const binaries = await this.deleteStaleBinaries()
+    this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
+
+    if (!this.dryRun) {
+      const budget = Date.now() + gcConfig.deleteBudgetMs
+      const parts = await this.dropCopiedParts(budget)
+      const single = await this.deleteSingleArticleBinaries(budget)
+      const releases = await this.deleteEmptyReleases(budget)
+      if (parts || single || releases) {
+        this.logger.info(
+          `Space: ${parts} copied parts, ${single} single-article binaries, ${releases} empty releases deleted.`
+        )
+      }
+    }
+
+    const deferred = await this.deleteOldDeferredFiles()
+    if (deferred) this.logger.info(`Deferred files: ${this.verb('delete', 'deleted')} ${deferred} past retention.`)
+
+    const { expired, scanned } = await this.expireLegacyPendingFiles()
+    if (scanned > 0) {
+      this.logger.info(
+        `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
+      )
+    }
+  }
+
+  /**
+   * Drops the oldest waiting body jobs above gcConfig.bodyQueueMax. BullMQ
+   * workers take jobs from the right end of the wait list, so the oldest are
+   * there.
+   */
+  private async trimBodyQueue(): Promise<number> {
+    const redis = RedisService.client
+    const key = QueueService.bodyQueue.toKey('wait')
+    const excess = (await redis.llen(key)) - gcConfig.bodyQueueMax
+    if (excess <= 0 || this.dryRun) return Math.max(excess, 0)
+    let dropped = 0
+    while (dropped < excess) {
+      const ids: string[] = (await redis.rpop(key, Math.min(5000, excess - dropped))) ?? []
+      if (ids.length === 0) break
+      await redis.unlink(...ids.map((id) => QueueService.bodyQueue.toKey(id)))
+      dropped += ids.length
+    }
+    return dropped
+  }
+
+  /**
+   * Deletes binaries (and, by cascade, their parts) that received no segment
+   * for `staleAfterSeconds`: incomplete ones are given up on; completed ones
+   * already live on in `files`; conflicted ones will never become a file.
+   * Completed ones still waiting for worker:file are kept.
+   */
+  private async deleteStaleBinaries(): Promise<number> {
+    const query = db
+      .from('binaries')
+      .where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
+      .where((q) =>
+        q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
+      )
+
+    if (this.dryRun) {
+      const [{ count }] = await query.count('* as count')
+      return Number(count)
+    }
+    return Number(await query.delete())
+  }
+
+  private async deleteOldDeferredFiles(): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.deferredRetentionDays * 24 * 60 * 60 * 1000)
+    let total = 0
+    for (const [table, column] of [['deferred_files', 'first_seen_at'], ['deferred_ranges', 'created_at']]) {
+      const query = db.from(table).where(column, '<', cutoff)
+      if (this.dryRun) {
+        const [{ count }] = await query.count('* as count')
+        total += Number(count)
+      } else {
+        total += Number(await query.delete())
+      }
+    }
+    return total
+  }
+
+  // Keyset cursors over the tables, kept between sweeps so one pass over
+  // millions of rows is spread over several sweeps.
+  private singleCursor = 0
+  private releaseCursor = 0
+
+  /** Parts of header-derived binaries that are a file already (worker:file now drops them itself). */
+  private async dropCopiedParts(budget: number): Promise<number> {
+    let total = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM parts WHERE binary_id IN (
+           SELECT b.id FROM binaries b
+           WHERE b.file_id IS NOT NULL AND b.size IS NULL
+             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id)
+           LIMIT ?)`,
+        [500]
+      )
+      total += result.rowCount ?? 0
+      if (!result.rowCount) break
+    }
+    return total
+  }
+
+  /** Header-derived binaries that never got a second article (gcConfig.binaries.singleArticleAfterSeconds). */
+  private async deleteSingleArticleBinaries(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.binaries.singleArticleAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const { rows } = await db.rawQuery(
+        `WITH batch AS (
+           SELECT id FROM binaries WHERE id > ? ORDER BY id LIMIT ?
+         ), gone AS (
+           DELETE FROM binaries b USING batch
+           WHERE b.id = batch.id AND b.completed_at IS NULL AND b.size IS NULL AND b.total_parts > 1
+             AND b.last_seen_at < ? AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) <= 1
+           RETURNING 1)
+         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
+        [this.singleCursor, gcConfig.deleteBatch, cutoff]
+      )
+      total += Number(rows[0].deleted)
+      if (rows[0].last === null) {
+        this.singleCursor = 0
+        break
+      }
+      this.singleCursor = Number(rows[0].last)
+    }
+    return total
+  }
+
+  /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
+  private async deleteEmptyReleases(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const { rows } = await db.rawQuery(
+        `WITH batch AS (
+           SELECT id FROM releases WHERE id > ? ORDER BY id LIMIT ?
+         ), gone AS (
+           DELETE FROM releases r USING batch
+           WHERE r.id = batch.id AND r.file_count = 0 AND r.first_seen_at < ?
+             AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
+             AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)
+             AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)
+             AND NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)
+           RETURNING 1)
+         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
+        [this.releaseCursor, gcConfig.deleteBatch, cutoff]
+      )
+      total += Number(rows[0].deleted)
+      if (rows[0].last === null) {
+        this.releaseCursor = 0
+        break
+      }
+      this.releaseCursor = Number(rows[0].last)
+    }
+    return total
+  }
+
+  /** Removes jobs in `state` that finished more than `ageSeconds` ago. */
+  private async cleanJobs(queue: Queue, state: 'completed' | 'failed', ageSeconds: number) {
+    const graceMs = ageSeconds * 1000
+    if (this.dryRun) {
+      const cutoff = Date.now() - graceMs
+      // Finished-job sets are sorted by finishedOn, so a score range counts them cheaply.
+      return RedisService.client.zcount(queue.toKey(state), '-inf', cutoff)
+    }
+
+    let total = 0
+    let removed: string[]
+    do {
+      removed = await queue.clean(graceMs, CLEAN_BATCH, state)
+      total += removed.length
+    } while (removed.length === CLEAN_BATCH)
+    return total
+  }
+
+  /**
+   * Pending-file hashes of the old header worker (and the pre-AdonisJS app)
+   * never expire. Give them a TTL rather than deleting them outright.
+   */
+  private async expireLegacyPendingFiles() {
+    const redis = RedisService.client
+    const { prefix, ttlSeconds } = gcConfig.legacyPendingFiles
+    let cursor = '0'
+    let scanned = 0
+    let expired = 0
+
+    do {
+      const [next, keys]: [string, string[]] = await redis.scan(
+        cursor,
+        'MATCH',
+        `${prefix}*`,
+        'COUNT',
+        SCAN_COUNT
+      )
+      cursor = next
+      if (keys.length === 0) continue
+      scanned += keys.length
+
+      const ttls: [Error | null, number][] = await keys
+        .reduce((pipeline, key) => pipeline.ttl(key), redis.pipeline())
+        .exec()
+      const orphaned = keys.filter((_, i) => ttls[i][1] === -1)
+      expired += orphaned.length
+
+      if (!this.dryRun && orphaned.length > 0) {
+        await orphaned
+          .reduce((pipeline, key) => pipeline.expire(key, ttlSeconds, 'NX'), redis.pipeline())
+          .exec()
+      }
+    } while (cursor !== '0')
+
+    return { scanned, expired }
+  }
+
+  private verb(present: string, past: string) {
+    return this.dryRun ? `would ${present}` : past
+  }
+}

commands/GroupAdd.ts

@@ -0,0 +1,72 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import Group from '#models/group'
+import NntpService from '#services/NntpService'
+import { parseGroupResponse } from '#lib/nntp/group'
+
+/**
+ * Adds newsgroups to index (or reactivates them). The scheduler picks them
+ * up on its next tick, starting INDEX_INITIAL_BACKFILL articles back.
+ */
+export default class GroupAdd extends BaseCommand {
+  public static commandName = 'group:add'
+  public static description = 'Adds newsgroups to index.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Group names, e.g. alt.binaries.hdtv' })
+  declare names: string[]
+
+  @flags.boolean({ description: 'Deactivate the groups instead' })
+  declare disable: boolean
+
+  @flags.boolean({
+    description: 'Also download bodies to identify obfuscated posts (one body per article; bandwidth-heavy)',
+  })
+  declare bodies: boolean
+
+  @flags.number({ description: 'Start this many articles back instead of INDEX_INITIAL_BACKFILL (new groups only)' })
+  declare backfill?: number
+
+  @flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' })
+  declare reset: boolean
+
+  public async run() {
+    for (const name of this.names) {
+      if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) {
+        this.logger.error(`Not a valid group name: ${name}`)
+        continue
+      }
+      const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies })
+      group.active = !this.disable
+      if (this.bodies) group.fetchBodies = true
+      if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
+        // The scheduler continues after last_queued_id.
+        const last = await this.lastArticle(name)
+        group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n)
+        group.lastIndexedId = group.lastQueuedId
+      }
+      await group.save()
+      this.logger.info(
+        `${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` +
+          (this.backfill !== undefined ? `, starting after article ${group.lastQueuedId}` : '')
+      )
+    }
+    if (this.backfill !== undefined) await NntpService.shutdown()
+  }
+
+  private async lastArticle(name: string): Promise<bigint> {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      return parseGroupResponse((await conn.group(name)).resp).last
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+}

commands/GroupsTop.ts

@@ -0,0 +1,176 @@
+import { writeFile } from 'node:fs/promises'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import NntpService from '#services/NntpService'
+import indexerConfig from '#config/indexer'
+import { parseGroupResponse } from '#lib/nntp/group'
+import { parseSubject } from '#lib/subject/index'
+
+interface GroupRate {
+  name: string
+  perMinute: number
+  sampled?: number
+  normal?: number
+  obfuscated?: number
+  skipped?: number
+  avgArticleBytes?: number
+  avgHeaderBytes?: number
+}
+
+/**
+ * Lists the busiest newsgroups (by articles posted per minute, from two
+ * LIST ACTIVE snapshots) and optionally samples their newest headers to
+ * estimate what indexing them would cost. See LOG.md (2026-09-28 (14)).
+ *
+ *   node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt
+ */
+export default class GroupsTop extends BaseCommand {
+  public static commandName = 'groups:top'
+  public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' })
+  declare pattern: string
+
+  @flags.number({ description: 'Minutes between the two snapshots', default: 5 })
+  declare minutes: number
+
+  @flags.number({ description: 'Number of groups to list', default: 1000 })
+  declare limit: number
+
+  @flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 })
+  declare sample: number
+
+  @flags.string({ description: 'Write the list to this file (default: print only)' })
+  declare out?: string
+
+  public async run() {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      this.logger.info(`Snapshot 1 of ${this.pattern}…`)
+      const first = await this.snapshot(conn)
+      this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`)
+      const started = Date.now()
+      await sleep(this.minutes * 60_000)
+      const second = await this.snapshot(conn)
+      const minutes = (Date.now() - started) / 60_000
+
+      const groups: GroupRate[] = []
+      for (const [name, high] of second) {
+        const before = first.get(name)
+        if (before === undefined || high <= before) continue
+        groups.push({ name, perMinute: Number(high - before) / minutes })
+      }
+      groups.sort((a, b) => b.perMinute - a.perMinute)
+      const top = groups.slice(0, this.limit)
+      this.logger.info(`${groups.length} active groups; listing ${top.length}.`)
+
+      if (this.sample > 0) {
+        for (const [i, group] of top.entries()) {
+          try {
+            Object.assign(group, await this.sampleGroup(conn, group.name))
+          } catch (error: any) {
+            this.logger.warning(`${group.name}: ${error.message}`)
+          }
+          if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`)
+        }
+      }
+
+      const text = this.format(top, minutes)
+      if (this.out) {
+        await writeFile(this.out, text)
+        this.logger.info(`Written to ${this.out}`)
+      } else {
+        console.log(text)
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+      await NntpService.shutdown()
+    }
+  }
+
+  /** Group name → highest article number, from LIST ACTIVE. */
+  private async snapshot(conn: any): Promise<Map<string, bigint>> {
+    const { data } = await conn.list(this.pattern)
+    const result = new Map<string, bigint>()
+    for (const line of Buffer.from(data).toString('latin1').split('\r\n')) {
+      const [name, high] = line.split(' ')
+      if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high))
+    }
+    return result
+  }
+
+  /** Classifies the newest headers the way worker:fetch does. */
+  private async sampleGroup(conn: any, name: string) {
+    const { last } = parseGroupResponse((await conn.group(name)).resp)
+    const start = last - BigInt(this.sample) + 1n
+    const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString())
+    let normal = 0
+    let skipped = 0
+    let obfuscated = 0
+    let obfuscatedBytes = 0
+    let headerBytes = 0
+    for (const [, header] of overviews as [number, Record<string, string>][]) {
+      if (!header) continue
+      headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0)
+      if (header.subject && parseSubject(header.subject)) normal++
+      else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
+      else {
+        obfuscated++
+        obfuscatedBytes += Number(header[':bytes']) || 0
+      }
+    }
+    const sampled = normal + skipped + obfuscated
+    return {
+      sampled,
+      normal,
+      skipped,
+      obfuscated,
+      avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0,
+      avgHeaderBytes: sampled ? headerBytes / sampled : 0,
+    }
+  }
+
+  private format(groups: GroupRate[], minutes: number): string {
+    const sampled = groups.some((g) => g.sampled)
+    const lines = [
+      `# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`,
+      sampled
+        ? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s'
+        : '# rank\tgroup\tarticles/min',
+    ]
+    let bodyTotal = 0
+    let headerTotal = 0
+    let articlesTotal = 0
+    for (const [i, g] of groups.entries()) {
+      articlesTotal += g.perMinute
+      if (!g.sampled) {
+        lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`)
+        continue
+      }
+      const obfuscatedShare = g.obfuscated! / g.sampled
+      // Bodies: every obfuscated article is downloaded once. Headers: every article.
+      const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6
+      const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6
+      bodyTotal += bodyMbit
+      headerTotal += headerMbit
+      lines.push(
+        [i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t')
+      )
+    }
+    lines.push(`# total: ${Math.round(articlesTotal)} articles/min`)
+    if (sampled) {
+      lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`)
+      lines.push(`#   of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`)
+    }
+    return lines.join('\n') + '\n'
+  }
+}

commands/IndexScheduler.ts

@@ -3,6 +3,10 @@ import { CommandOptions } from '@adonisjs/core/types/ace'
 import Group from '#models/group'
 import NntpService from '#services/NntpService'
 import QueueService from '#services/QueueService'
+import indexerConfig from '#config/indexer'
+import Shutdown from '#services/Shutdown'
+import FetchRangeService from '#services/FetchRangeService'
+import SplitService from '#services/SplitService'
 
 export default class IndexScheduler extends BaseCommand {
   public static commandName = 'index:scheduler'
@@ -13,7 +17,6 @@ export default class IndexScheduler extends BaseCommand {
   }
 
   private pool = NntpService
-  private fetchQueue = QueueService.nntpFetchQueue
 
   public async run() {
     this.logger.info('Scheduler started. Awaiting tasks...')
@@ -28,58 +31,89 @@ export default class IndexScheduler extends BaseCommand {
       }
 
       let conn
+      let failed = false
       try {
         conn = await this.pool.acquire()
 
         for (const group of groups) {
           try {
-            const groupInfo: any = await conn.group(group.name)
-            
-            // nntp-js returns article numbers as strings. We must parse them to BigInts.
-            const firstArticle = BigInt(groupInfo.first)
-            const lastArticle = BigInt(groupInfo.last)
+            const { first: firstArticle, last: lastArticle } = await this.pool.selectGroup(
+              conn,
+              group.name
+            )
 
-            // lastIndexedId from the database should also be treated as a BigInt.
-            const lastIndexed = group.lastIndexedId ? BigInt(group.lastIndexedId) : null
+            await FetchRangeService.requeue(group)
 
-            const startId = lastIndexed ? lastIndexed + 1n : firstArticle
+            // Postgres returns bigint columns as strings.
+            const lastQueued = BigInt(group.lastQueuedId ?? 0)
+            const { batchSize, maxBatchesPerTick, initialBackfill } = indexerConfig
+            let startId =
+              lastQueued === 0n ? lastArticle - initialBackfill + 1n : lastQueued + 1n
+            if (startId < firstArticle) startId = firstArticle
 
             if (startId > lastArticle) {
+              await FetchRangeService.advanceWatermark(group)
               this.logger.info(`No new headers for group ${group.name}.`)
               continue
             }
 
-            const BATCH_SIZE = 100000n
-            const proposedEndId = startId + BATCH_SIZE - 1n
-            const endId = proposedEndId < lastArticle ? proposedEndId : lastArticle
+            let batches = 0
+            let endId = startId - 1n
+            while (endId < lastArticle && batches < maxBatchesPerTick) {
+              const batchStart = endId + 1n
+              endId = batchStart + batchSize - 1n < lastArticle ? batchStart + batchSize - 1n : lastArticle
+              await FetchRangeService.queue(group, batchStart, endId)
+              batches++
+            }
 
-            this.logger.info(`Queueing fetch job for ${group.name}: articles ${startId} to ${endId}`)
-            await this.fetchQueue.add('fetch-headers', {
-              groupName: group.name,
-              startId: startId.toString(),
-              endId: endId.toString(),
-            })
+            this.logger.info(
+              `Queued ${batches} fetch job(s) for ${group.name}: articles ${startId} to ${endId}` +
+                (endId < lastArticle ? ` (${lastArticle - endId} left for next tick)` : '')
+            )
 
-            group.lastIndexedId = endId
+            group.lastQueuedId = endId
             await group.save()
+            const watermark = await FetchRangeService.advanceWatermark(group)
+            if (endId - watermark > batchSize * BigInt(maxBatchesPerTick) * 3n) {
+              this.logger.warning(
+                `${group.name}: indexed up to ${watermark}, ${endId - watermark} articles behind the queue.`
+              )
+            }
           } catch (err: any) {
+            failed = true
             this.logger.error(`Error processing group ${group.name}: ${err.message}`)
           }
         }
       } catch (err: any) {
+        failed = true
         this.logger.error(`Error in scheduler main loop: ${err.message}`)
       } finally {
         if (conn) {
-          this.pool.release(conn)
+          this.pool.release(conn, failed)
         }
       }
     }
 
-    // Run once immediately and then on an interval.
-    schedule()
-    setInterval(schedule, 60000) // 1 minute
+    const split = async () => {
+      try {
+        const result = await SplitService.splitPending()
+        if (result.parents > 0) {
+          this.logger.info(
+            `Split ${result.parents} conflicted binaries: ${result.volumes} verified volumes, ${result.completed} completed.`
+          )
+        }
+      } catch (err: any) {
+        this.logger.error(`Error splitting conflicted binaries: ${err.message}`)
+      }
+    }
 
-    // Keep the command running
-    await new Promise(() => {})
+    // Run immediately, then wait a full interval after each run so runs never overlap.
+    while (!Shutdown.requested) {
+      await schedule()
+      await split()
+      await Shutdown.sleep(indexerConfig.intervalMs)
+    }
+    await this.pool.shutdown()
+    await QueueService.closeAll()
   }
 }

commands/ReleaseRebuild.ts

@@ -0,0 +1,163 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import IngestService from '#services/IngestService'
+import NameService from '#services/NameService'
+import QueueService from '#services/QueueService'
+import { rarReleaseIdentity, releaseIdentity } from '#lib/release/index'
+
+const BATCH = 5000
+
+/**
+ * Recomputes the release of every binary (and its file) with the current
+ * rules in lib/release. Used after changing those rules.
+ */
+export default class ReleaseRebuild extends BaseCommand {
+  public static commandName = 'release:rebuild'
+  public static description = 'Reassigns all binaries and files to releases using the current grouping rules.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({
+    description:
+      'Also drop everything derived from file contents (par2 lists, par2/RAR groupings, real names), ' +
+      'to rebuild it with content:backfill --all',
+  })
+  declare resetContent: boolean
+
+  public async run() {
+    if (this.resetContent) await this.dropContentDerived()
+
+    let lastId = 0
+    let assigned = 0
+    let unassigned = 0
+    try {
+      while (true) {
+        const binaries = await db
+          .from('binaries')
+          .where('id', '>', lastId)
+          .orderBy('id')
+          .limit(BATCH)
+          .select('id', 'poster', 'subject', 'name', 'groups', 'posted_at')
+        if (binaries.length === 0) break
+        lastId = Number(binaries.at(-1).id)
+
+        const segments = binaries.map((binary) => {
+          // Body-derived binaries store a synthetic subject, see segmentFromYenc.
+          const weak = /^"[^"]*" yEnc$/.test(binary.subject)
+          return {
+            id: Number(binary.id),
+            poster: binary.poster,
+            groups: binary.groups,
+            postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
+            weakIdentity: weak,
+            release: releaseIdentity({ poster: binary.poster, subject: binary.subject, name: binary.name, weak }),
+          }
+        })
+
+        await db.transaction(async (trx) => {
+          const releaseIds = await IngestService.upsertReleases(trx.knexClient, segments)
+          const ids = segments.map((s) => s.id)
+          const releaseIdsPerBinary = segments.map((s) =>
+            s.release ? releaseIds.get(s.release.key)! : null
+          )
+          await trx.rawQuery(
+            `UPDATE binaries b SET release_id = v.release_id
+             FROM unnest(?::bigint[], ?::bigint[]) AS v(id, release_id)
+             WHERE b.id = v.id AND b.release_id IS DISTINCT FROM v.release_id
+               AND NOT (v.release_id IS NULL
+                        AND b.release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))
+               -- A posted NZB decided this grouping (NzbImportService); name rules don't override it.
+               AND NOT EXISTS (SELECT 1 FROM files n WHERE n.release_id = b.release_id AND n.content_type = 'nzb')`,
+            [ids, releaseIdsPerBinary] as any
+          )
+        })
+        assigned += segments.filter((s) => s.release).length
+        unassigned += segments.filter((s) => !s.release).length
+      }
+
+      // Files grouped by their RAR headers: regroup from the stored listing (no downloads).
+      let rarGrouped = 0
+      let rarCursor = 0
+      while (true) {
+        const rows = await db
+          .from('files as f')
+          .join('binaries as b', 'b.file_id', 'f.id')
+          .whereNotNull('f.archive')
+          .whereNull('b.release_id')
+          .where('f.id', '>', rarCursor)
+          .orderBy('f.id')
+          .limit(BATCH)
+          .select('f.id as file_id', 'f.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
+        if (rows.length === 0) break
+        rarCursor = Number(rows.at(-1).file_id)
+        const segments = rows.map((r) => ({
+          fileId: Number(r.file_id),
+          binaryId: Number(r.binary_id),
+          poster: r.poster,
+          groups: r.groups,
+          postedAt: r.posted_at ? new Date(r.posted_at) : undefined,
+          weakIdentity: true,
+          release: rarReleaseIdentity(r.archive?.entries ?? []),
+        }))
+        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), segments)
+        for (const s of segments.filter((x) => x.release)) {
+          const releaseId = ids.get(s.release!.key)
+          await db.from('binaries').where('id', s.binaryId).update({ release_id: releaseId })
+          await db.from('files').where('id', s.fileId).update({ release_id: releaseId })
+          rarGrouped++
+        }
+      }
+
+      // Real names from par2 for binaries with a known size (no downloads).
+      let named = 0
+      let nameCursor = 0
+      while (true) {
+        const rows = await db.from('binaries').whereNotNull('size').where('id', '>', nameCursor).orderBy('id').limit(BATCH).select('id')
+        if (rows.length === 0) break
+        nameCursor = Number(rows.at(-1).id)
+        named += await NameService.resolve(rows.map((r) => Number(r.id)))
+      }
+
+      const files = await db.rawQuery(
+        `UPDATE files f SET release_id = b.release_id FROM binaries b
+         WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`
+      )
+      const orphans = await db.rawQuery(
+        `DELETE FROM releases r WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
+           AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)`
+      )
+      if (this.resetContent) {
+        this.logger.info('Content-derived data dropped; run `node ace content:backfill --all` to rebuild it.')
+      }
+      this.logger.info(
+        `Binaries: ${assigned} in a release, ${unassigned} without, ${rarGrouped} regrouped from RAR headers, ` +
+          `${named} named from par2. ` +
+          `Files updated: ${files.rowCount}. Empty releases removed: ${orphans.rowCount}.`
+      )
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+
+  /** par2 lists, par2/RAR release assignments and recovered names. */
+  private async dropContentDerived() {
+    await db.transaction(async (trx) => {
+      await trx.rawQuery('DELETE FROM release_files')
+      await trx.rawQuery(`UPDATE binaries SET real_name = NULL WHERE real_name IS NOT NULL`)
+      await trx.rawQuery(`UPDATE files SET real_name = NULL, archive = NULL WHERE real_name IS NOT NULL OR archive IS NOT NULL`)
+      await trx.rawQuery(`UPDATE releases SET contents = NULL, encrypted = NULL WHERE contents IS NOT NULL OR encrypted IS NOT NULL`)
+      // Binaries whose name rules find no release were grouped from contents (or wrongly by size).
+      await trx.rawQuery(`
+        UPDATE binaries SET release_id = NULL
+        WHERE subject ~ '^"[^"]*" yEnc$' AND release_id IS NOT NULL
+          AND (name !~ '\\.[A-Za-z0-9]{1,5}$' OR release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))`)
+      await trx.rawQuery(`
+        UPDATE files f SET release_id = b.release_id FROM binaries b
+        WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`)
+    })
+    this.logger.info('Dropped par2 lists, content-derived groupings and recovered names.')
+  }
+}

commands/ReleaseRename.ts

@@ -0,0 +1,20 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import { renameFromStoredContents } from '#services/ArchiveService'
+import { regroupAndRenameFromPar2 } from '#services/NameService'
+
+/** Renames random-named releases from their stored archive listings (no downloads). */
+export default class ReleaseRename extends BaseCommand {
+  public static commandName = 'release:rename'
+  public static description = 'Renames random-named releases from their par2 listings and archive contents.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const par2 = await regroupAndRenameFromPar2()
+    this.logger.info(`par2: ${par2.regrouped} files moved into the set that names them, ${par2.renamed} releases renamed.`)
+    this.logger.info(`Renamed ${await renameFromStoredContents()} releases from their archive contents.`)
+  }
+}

commands/ReleaseStats.ts

@@ -0,0 +1,57 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+
+/** Prints how well files are being grouped into releases. */
+export default class ReleaseStats extends BaseCommand {
+  public static commandName = 'release:stats'
+  public static description = 'Shows release recognition statistics.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const { rows } = await db.rawQuery(`
+      WITH r AS (
+        SELECT r.id, r.file_total, r.poster IS NULL AS weak, r.source,
+               count(b.id) AS binaries,
+               count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed
+        FROM releases r LEFT JOIN binaries b ON b.release_id = r.id
+        GROUP BY r.id
+      )
+      SELECT
+        count(*) AS releases,
+        count(*) FILTER (WHERE completed > 0) AS with_files,
+        count(*) FILTER (WHERE completed >= 2) AS multi_file,
+        count(*) FILTER (WHERE file_total IS NOT NULL AND completed >= file_total) AS complete_by_counter,
+        count(*) FILTER (WHERE weak) AS from_obfuscated,
+        count(*) FILTER (WHERE weak AND completed > 0) AS from_obfuscated_with_files
+      FROM r`)
+    const { rows: coverage } = await db.rawQuery(`
+      SELECT count(*) AS completed_files,
+             count(*) FILTER (WHERE release_id IS NOT NULL) AS in_release
+      FROM binaries WHERE completed_at IS NOT NULL`)
+    const { rows: bySource } = await db.rawQuery(`
+      SELECT coalesce(r.source, 'unknown') AS source, count(DISTINCT r.id) AS releases,
+             count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed_files
+      FROM releases r JOIN binaries b ON b.release_id = r.id GROUP BY 1 ORDER BY 2 DESC`)
+    // Readable names: descriptive as posted, or recovered from contents.
+    const { rows: naming } = await db.rawQuery(`
+      WITH f AS (
+        SELECT real_name IS NOT NULL AS recovered,
+               is_readable_name(coalesce(real_name, filename)) AS readable_name,
+               regexp_replace(coalesce(real_name, filename),
+                 '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|mkv|mp4|avi|flac|mp3|jpg))+$', '', 'i') AS stem,
+               coalesce(real_name, filename) AS name
+        FROM files
+      )
+      SELECT count(*) AS files,
+             count(*) FILTER (WHERE recovered) AS names_recovered,
+             count(*) FILTER (WHERE name ~ '\\.[A-Za-z0-9]{1,5}$' AND readable_name) AS readable
+      FROM f`)
+    this.logger.info(JSON.stringify({ ...rows[0], ...coverage[0] }))
+    this.logger.info(`naming: ${JSON.stringify(naming[0])}`)
+    this.logger.info(`by source: ${JSON.stringify(bySource)}`)
+  }
+}

commands/workers/Body.ts

@@ -4,9 +4,13 @@ import { Worker } from 'bullmq'
 import fs from 'node:fs/promises'
 import path from 'node:path'
 import queueConfig from '#config/queue'
+import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
+import env from '#start/env'
 import NntpService from '#services/NntpService'
-import { parseYencMeta } from '#services/YencService'
+import IngestService, { segmentFromYenc } from '#services/IngestService'
+import { parseYencMeta, YencError } from '#lib/yenc/index'
+import { firstChunkMd5 } from '#services/ContentService'
 
 export default class BodyWorker extends BaseCommand {
   public static commandName = 'worker:body'
@@ -20,31 +24,26 @@ export default class BodyWorker extends BaseCommand {
     this.logger.info('Starting body worker...')
 
     const pool = NntpService
-    const headerQueue = QueueService.headerQueue
 
     const worker = new Worker('body-queue', async (job) => {
-      const { header, group } = job.data
+      const { header, group, articleNumber } = job.data
       const messageId = header['message-id']
       this.logger.debug(`Processing header with unparsable subject: ${header.subject}`)
 
       let conn
+      let failed = false
       try {
         conn = await pool.acquire()
-        const bodyBuffer: Buffer = (await conn.body(messageId)).data
+        const bodyBuffer = await pool.fetchBody(conn, messageId)
 
+        let meta
         try {
-          const meta = parseYencMeta(bodyBuffer)
-          if (meta.header.name) {
-            const { name, part, total } = meta.header
-            const newSubject = `"${name}" yEnc (${part}/${total})`
-            header.subject = newSubject
-
-            this.logger.info(`Found yEnc metadata in body. New subject: ${newSubject}`)
-            await headerQueue.add('process-header', { header, group })
-          } else {
-            this.logger.warning(`Could not find yEnc metadata in body for header: ${header.subject}`)
-          }
+          meta = parseYencMeta(bodyBuffer)
         } catch (parseError: any) {
+          if (parseError instanceof YencError && bodyBuffer.indexOf('=ybegin ') === -1) {
+            this.logger.debug(`Not a yEnc post, skipping: ${messageId}`)
+            return
+          }
           this.logger.error(`Failed to parse yEnc data for message ID ${messageId}. Dumping buffer.`)
           const debugDir = path.join(this.app.appRoot.pathname, 'debug')
           await fs.mkdir(debugDir, { recursive: true })
@@ -54,21 +53,45 @@ export default class BodyWorker extends BaseCommand {
           this.logger.error(`Problematic body buffer saved to: ${dumpFile}`)
           throw parseError
         }
+
+        pool.release(conn)
+        conn = undefined
+
+        const segment = segmentFromYenc(header, group, meta, articleNumber)
+        if (segment?.number === 1) segment.md5_16k = firstChunkMd5(bodyBuffer, meta.begin.size)
+        if (!segment) {
+          this.logger.warning(`Inconsistent yEnc part numbering in ${messageId}, skipping.`)
+          return
+        }
+        const completed = await IngestService.ingest([segment])
+        this.logger.debug(
+          `Stored part ${segment.number}/${segment.totalParts} of "${segment.name}"` +
+            (completed.length > 0 ? ' (file complete)' : '')
+        )
       } catch (error: any) {
+        failed = true
         this.logger.error(`Error in body worker for message ID ${messageId}: ${error.message}`)
         throw error
       } finally {
         if (conn) {
-          pool.release(conn)
+          pool.release(conn, failed)
         }
       }
-    }, { connection: queueConfig.connection })
+    }, {
+      connection: queueConfig.connection,
+      // Each job holds one NNTP connection while downloading, so keep this <= the pool size.
+      concurrency: env.get('BODY_CONCURRENCY', 1),
+    })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`Body job ${job?.id} failed: ${err.message}`)
     })
 
     this.logger.info('Body worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
   }
 }

commands/workers/Collection.ts

@@ -2,10 +2,12 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import queueConfig from '#config/queue'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import QueueService from '#services/QueueService'
 import NntpService from '#services/NntpService'
-import { YencFile } from '#services/YencFile'
+import ContentService from '#services/ContentService'
 import File from '#models/file'
-import { createExtractorFromData } from 'node-unrar-js'
 
 export default class CollectionWorker extends BaseCommand {
   public static commandName = 'worker:collection'
@@ -18,8 +20,6 @@ export default class CollectionWorker extends BaseCommand {
   public async run() {
     this.logger.info('Starting collection worker...')
 
-    const pool = NntpService
-
     const worker = new Worker('collection-queue', async (job) => {
       const { fileId } = job.data
       this.logger.debug(`Processing file ID ${fileId} for collection.`)
@@ -30,51 +30,47 @@ export default class CollectionWorker extends BaseCommand {
         return
       }
 
-      const RAR_REGEX = /\.part0*1\.rar$/
-      if (RAR_REGEX.test(file.filename)) {
-        this.logger.info(`File "${file.filename}" is the first part of a RAR set.`)
-
-        const firstPart = file.messageIds['1']
-        if (!firstPart || !firstPart.id) {
-          this.logger.error(`Could not find message ID for the first part of file "${file.filename}".`)
-          return
+      try {
+        const content = await ContentService.inspect(fileId)
+        if (content?.par2) {
+          const { par2 } = content
+          this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
+        } else if (content?.archive) {
+          const { info, releaseRenamed, grouped } = content.archive
+          const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
+            .filter(Boolean)
+            .join(', ')
+          this.logger.info(
+            `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
+              (grouped ? ', grouped into a release' : '') +
+              (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
+          )
         }
-
-        let conn
-        try {
-          conn = await pool.acquire()
-          const bodyBuffer = (await conn.body(`<${firstPart.id}>`)).data
-
-          const yencFile = new YencFile()
-          yencFile.processPart(bodyBuffer)
-          const decodedBuffer = yencFile.getBuffer()
-
-          const extractor = await createExtractorFromData({ data: new Uint8Array(decodedBuffer).buffer })
-          const fileList = extractor.getFileList()
-          // In a real implementation, we would save this file list.
-          this.logger.info(`Files in "${file.filename}": ${JSON.stringify(fileList)}`)
-
-        } catch (error: any) {
-          if (error.code === 430) {
-            this.logger.error(`Article not found for first part of RAR set (Message ID: ${firstPart.id})`)
-          } else {
-            this.logger.error(`Error processing RAR file: ${error.message}`)
-          }
-        } finally {
-          if (conn) {
-            pool.release(conn)
-          }
+        if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
+        if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
+        if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
+        if (content?.nzb) {
+          const n = content.nzb
+          this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
         }
-      } else {
-        this.logger.debug(`File "${file.filename}" is not the first part of a RAR set.`)
+      } catch (error: any) {
+        this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
       }
-    }, { connection: queueConfig.connection })
+    }, {
+      connection: queueConfig.connection,
+      // Each job holds one NNTP connection while downloading; keep <= the pool size.
+      concurrency: env.get('COLLECTION_CONCURRENCY', 1),
+    })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
     })
 
     this.logger.info('Collection worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
   }
 }

commands/workers/Deferred.ts

@@ -0,0 +1,57 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import DeferredService from '#services/DeferredService'
+
+const IDLE_MS = 60_000
+
+/**
+ * Works through `deferred_files` (CONCEPT.md D28): one body per file, and
+ * small files completely. Not started by default (compose profile
+ * `deferred`); run it when there is bandwidth to spare.
+ */
+export default class DeferredWorker extends BaseCommand {
+  public static commandName = 'worker:deferred'
+  public static description = 'Samples deferred files (e.g. ngPost uploads): one article body per file.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Stop after this many files (default: keep running)' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = env.get('DEFERRED_CONCURRENCY', 4)
+    this.logger.info(`Deferred worker started (concurrency ${concurrency}).`)
+    const counts: Record<string, number> = {}
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        if (batch.length === 0) {
+          if (this.limit) break
+          await Shutdown.sleep(IDLE_MS)
+          continue
+        }
+        let next = 0
+        await Promise.all(
+          Array.from({ length: concurrency }, async () => {
+            while (next < batch.length) {
+              const status = await DeferredService.process(batch[next++])
+              counts[status] = (counts[status] ?? 0) + 1
+            }
+          })
+        )
+        done += batch.length
+        this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+      }
+    } finally {
+      await NntpService.shutdown()
+      await QueueService.closeAll()
+    }
+  }
+}

commands/workers/Fetch.ts

@@ -2,12 +2,20 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import queueConfig from '#config/queue'
+import env from '#start/env'
+import db from '@adonisjs/lucid/services/db'
+import indexerConfig from '#config/indexer'
+import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
 import NntpService from '#services/NntpService'
+import FetchRangeService from '#services/FetchRangeService'
+import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
+import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService'
+import SpotService, { isSpotGroup } from '#services/SpotService'
 
 export default class FetchWorker extends BaseCommand {
   public static commandName = 'worker:fetch'
-  public static description = 'Starts a worker to fetch headers from the NNTP server.'
+  public static description = 'Starts a worker that fetches headers (XOVER) and stores them as binaries/parts.'
 
   public static options: CommandOptions = {
     startApp: true,
@@ -17,53 +25,86 @@ export default class FetchWorker extends BaseCommand {
     this.logger.info('Starting fetch worker...')
 
     const pool = NntpService
-    const headerQueue = QueueService.headerQueue
+    const bodyQueue = QueueService.bodyQueue
 
     const worker = new Worker('nntp-fetch-queue', async (job) => {
       const { groupName, startId, endId } = job.data
       this.logger.info(`Processing fetch job for ${groupName}, articles ${startId}-${endId}`)
 
+      let overviews: [number, Record<string, string>][]
       let conn
+      let failed = false
       try {
         conn = await pool.acquire()
         await conn.group(groupName)
-        const overview: any = await conn.xover(startId, endId)
-        this.logger.info(`Fetched ${overview.overviews.length} headers from ${groupName}.`)
-
-        if (overview.overviews.length > 0) {
-          const jobs = overview.overviews.map(([id, header]: [number, any]) => {
-            if (!header) {
-              this.logger.warning(`Header is undefined for job ${job.id}`)
-              return null
-            }
-            return {
-              name: 'process-header',
-              data: { header, group: groupName },
-              opts: { jobId: `${groupName}-${id}` },
-            }
-          }).filter(Boolean)
-
-          await headerQueue.addBulk(jobs)
-          this.logger.info(`Added ${jobs.length} header jobs to the queue for ${groupName}.`)
-        }
+        overviews = (await conn.xover(startId, endId)).overviews
       } catch (error: any) {
+        failed = true
         this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`)
         throw error
       } finally {
         if (conn) {
-          pool.release(conn)
+          pool.release(conn, failed)
         }
       }
-    }, { 
+
+      if (isSpotGroup(groupName)) {
+        const spots = await SpotService.recordHeaders(groupName, overviews)
+        if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
+        this.logger.info(`${groupName} ${startId}-${endId}: ${overviews.length} headers, ${spots} spots queued.`)
+        return
+      }
+
+      const group = await db.from('groups').where('name', groupName).first()
+      const fetchBodies = Boolean(group?.fetch_bodies)
+
+      const segments: IngestSegment[] = []
+      const obfuscated: { articleNumber: number; header: Record<string, string> }[] = []
+      const deferred: DeferredArticle[] = []
+      let skipped = 0
+      for (const [articleNumber, header] of overviews) {
+        if (!header) continue
+        const segment = segmentFromHeader(header, groupName, articleNumber)
+        if (segment) segments.push(segment)
+        // Recorded per file (or as article ranges) in every group, bodies or not; processed later if at all.
+        else if (isDeferred(header)) deferred.push({ articleNumber, header })
+        else if (!fetchBodies || indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
+        else obfuscated.push({ articleNumber, header })
+      }
+
+      const completed = await IngestService.ingest(segments)
+      const deferredFiles = await DeferredService.record(groupName, deferred)
+
+      if (obfuscated.length > 0) {
+        await bodyQueue.addBulk(obfuscated.map(({ articleNumber, header }) => bodyJob(header, groupName, articleNumber)))
+      }
+
+      if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
+
+      this.logger.info(
+        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${segments.length} stored, ` +
+          `${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` +
+          `${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.`
+      )
+    }, {
       connection: queueConfig.connection,
-      concurrency: 5,
+      // XOVER is quick; most of a job is storing. Each concurrent job uses one DB connection (DB_POOL_MAX).
+      concurrency: env.get('FETCH_CONCURRENCY', 5),
     })
 
-    worker.on('failed', (job, err) => {
+    worker.on('failed', async (job, err) => {
       this.logger.error(`Fetch job ${job?.id} failed for group ${job?.data.groupName}: ${err.message}`)
+      // Only after the last attempt; the scheduler re-queues the range later.
+      if (job?.data.rangeId && job.attemptsMade >= (job.opts.attempts ?? 1)) {
+        await FetchRangeService.markFailed(job.data.rangeId, err.message).catch(() => {})
+      }
     })
 
     this.logger.info('Fetch worker started and listening for jobs.')
-    await new Promise(() => {}) // Keep command running
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
   }
 }

commands/workers/File.ts

@@ -1,13 +1,14 @@
 import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
+import db from '@adonisjs/lucid/services/db'
 import queueConfig from '#config/queue'
+import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
-import File from '#models/file'
 
 export default class FileWorker extends BaseCommand {
   public static commandName = 'worker:file'
-  public static description = 'Starts a worker to process completed files.'
+  public static description = 'Starts a worker that turns completed binaries into files.'
 
   public static options: CommandOptions = {
     startApp: true,
@@ -19,45 +20,78 @@ export default class FileWorker extends BaseCommand {
     const collectionQueue = QueueService.collectionQueue
 
     const worker = new Worker('file-queue', async (job) => {
-      const { filename, parts, groups } = job.data
-      const partCount = Object.keys(parts).length
-      this.logger.debug(`Processing complete file: "${filename}" with ${partCount} parts.`)
+      const { binaryId } = job.data
 
-      const firstPart = JSON.parse(Object.values(parts)[0] as string)
-      const poster = firstPart.from
-      const date = new Date(firstPart.date).getTime()
-
-      const messageIds = Object.entries(parts).reduce((acc, [partNumber, partData]) => {
-        const part = JSON.parse(partData as string)
-        const messageId = part['message-id']
-        if (messageId) {
-          acc[partNumber] = {
-            id: messageId.replace(/[<>]/g, ''),
-            size: part[':bytes'],
-          }
-        } else {
-          this.logger.warning(`Message ID not found for part ${partNumber} of file "${filename}"`)
-        }
-        return acc
-      }, {} as Record<string, any>)
-
-      if (Object.keys(messageIds).length !== partCount) {
-        throw new Error(`Could not process all parts for file "${filename}" due to missing message IDs.`)
+      const binary = await db.from('binaries').where('id', binaryId).first()
+      if (!binary) {
+        this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`)
+        return
       }
 
-      const file = await File.create({
-        filename,
-        poster,
-        date,
-        parts: partCount,
-        messageIds,
-        groups,
-      })
+      if (binary.conflicted_at) {
+        this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`)
+        return
+      }
 
-      this.logger.debug(`Saved file "${filename}" to database with ID: ${file.id}`)
+      const parts = await db
+        .from('parts')
+        .where('binary_id', binaryId)
+        .orderBy('number')
+        .select('number', 'message_id', 'bytes')
+      if (parts.length < binary.total_parts) {
+        throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
+      }
 
-      await collectionQueue.add('process-collection', { fileId: file.id })
-      this.logger.debug(`Added file ID ${file.id} to collection queue.`)
+      const messageIds = Object.fromEntries(
+        parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }])
+      )
+
+      // Already known as a file, e.g. added from a posted NZB before its own articles completed.
+      const existing = await db
+        .from('files')
+        .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? ''])
+        .first()
+      if (existing) {
+        await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
+        await dropCopiedParts(binary)
+        this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`)
+        return
+      }
+
+      // file_key is unique: a second completion of the same upload inserts nothing.
+      const knex = db.connection().getWriteClient()
+      const now = new Date()
+      const [inserted] = await knex('files')
+        .insert({
+          file_key: binary.file_key,
+          filename: binary.name,
+          poster: binary.poster,
+          date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
+          parts: parts.length,
+          message_ids: JSON.stringify(messageIds),
+          groups: JSON.stringify(binary.groups),
+          release_id: binary.release_id,
+          real_name: binary.real_name,
+          size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
+          created_at: now,
+          updated_at: now,
+        })
+        .onConflict('file_key')
+        .ignore()
+        .returning('id')
+
+      const fileId =
+        inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
+      await db.from('binaries').where('id', binaryId).update({ file_id: fileId })
+      await dropCopiedParts(binary)
+
+      if (!inserted) {
+        this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`)
+        return
+      }
+
+      this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
+      await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
     }, { connection: queueConfig.connection })
 
     worker.on('failed', (job, err) => {
@@ -65,6 +99,19 @@ export default class FileWorker extends BaseCommand {
     })
 
     this.logger.info('File worker started and listening for jobs.')
-    await new Promise(() => {})
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await QueueService.closeAll()
   }
 }
+
+/**
+ * The file holds every message id now. Binaries identified by their subject
+ * (no size: header-derived) don't need their parts any more; body-derived
+ * ones keep them, so articles of an equal-sized next volume are recognised
+ * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
+ */
+async function dropCopiedParts(binary: { id: number; size: number | null }) {
+  if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete()
+}

commands/workers/Header.ts

@@ -1,65 +0,0 @@
-import { BaseCommand } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import { Worker } from 'bullmq'
-import queueConfig from '#config/queue'
-import QueueService from '#services/QueueService'
-import RedisService from '#services/RedisService'
-
-export default class HeaderWorker extends BaseCommand {
-  public static commandName = 'worker:header'
-  public static description = 'Starts a worker to process headers.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  public async run() {
-    this.logger.info('Starting header worker...')
-
-    const redis = RedisService.client
-    const fileQueue = QueueService.fileQueue
-    const bodyQueue = QueueService.bodyQueue
-
-    const worker = new Worker('header-queue', async (job) => {
-      const { header, group } = job.data
-      if (!header || !header.subject) {
-        this.logger.warning(`Received job with invalid header data. JobID: ${job.id}`)
-        return
-      }
-
-      const subject = header.subject
-      const SUBJECT_REGEX = /"(.+)"(?: yEnc)? \((\d+)\/(\d+)\)/
-      const match = subject.match(SUBJECT_REGEX)
-
-      if (match) {
-        const filename = match[1]
-        const part = parseInt(match[2], 10)
-        const total = parseInt(match[3], 10)
-
-        const fileKey = `file:${filename}`
-        await redis.hset(fileKey, part, JSON.stringify(header))
-
-        const partCount = await redis.hlen(fileKey)
-
-        if (partCount === total) {
-          const fileParts = await redis.hgetall(fileKey)
-          await fileQueue.add('process-file', { filename, parts: fileParts, groups: [group] })
-          await redis.del(fileKey)
-          this.logger.info(`File "${filename}" is complete and moved to file-queue.`)
-        } else {
-          this.logger.info(`Stored part ${part}/${total} for file "${filename}"`)
-        }
-      } else {
-        this.logger.warning(`Could not parse subject: "${subject}". Moving to body-queue.`)
-        await bodyQueue.add('process-body', { header, group })
-      }
-    }, { connection: queueConfig.connection })
-
-    worker.on('failed', (job, err) => {
-      this.logger.error(`Header job ${job?.id} failed: ${err.message}`)
-    })
-    
-    this.logger.info('Header worker started and listening for jobs.')
-    await new Promise(() => {})
-  }
-}

commands/workers/Spots.ts

@@ -0,0 +1,54 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+const IDLE_MS = 30_000
+
+/**
+ * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
+ * downloads its compressed NZB (a few KB to a few MB) and imports it.
+ */
+export default class SpotsWorker extends BaseCommand {
+  public static commandName = 'worker:spots'
+  public static description = 'Reads Spotnet spots: verifies them and imports their NZBs.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Stop after this many spots (default: keep running)' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = env.get('SPOT_CONCURRENCY', 4)
+    this.logger.info(`Spot worker started (concurrency ${concurrency}).`)
+    const counts: Record<string, number> = {}
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const batch = await SpotService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        if (batch.length === 0) {
+          if (this.limit) break
+          await Shutdown.sleep(IDLE_MS)
+          continue
+        }
+        let next = 0
+        await Promise.all(
+          Array.from({ length: concurrency }, async () => {
+            while (next < batch.length) {
+              const status = await SpotService.process(batch[next++])
+              counts[status] = (counts[status] ?? 0) + 1
+            }
+          })
+        )
+        done += batch.length
+        this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}